From 7ff9e401025572f70cb2ded8ae330ecd1c19e275 Mon Sep 17 00:00:00 2001
From: yunus-dev-codecrafter <yunusabdulhameed1@gmail.com>
Date: Sat, 17 Oct 2026 17:30:29 +0100
Subject: [PATCH] test: add local generator performance benchmark harness

Measure all four bundled template generation paths through the trusted template pipeline with a bounded, offline harness that performs no dependency install, Git initialization, or network access. It reports deterministic environment/artifact metadata plus wall-clock statistics (mean, median, p95, min, max, standard deviation, coefficient of variation) so repeated runs on the same platform can be compared for regressions. Timings are not asserted in CI; the suite verifies coverage of every path, offline generation without node_modules, and structural reproducibility.
---
 docs/contributing/testing.md         |  22 ++
 tests/generator-benchmark.test.ts    | 167 +++++++++++++++
 tests/helpers/generator-benchmark.ts | 308 +++++++++++++++++++++++++++
 tests/helpers/index.ts               |  15 ++
 4 files changed, 512 insertions(+)
 create mode 100644 tests/generator-benchmark.test.ts
 create mode 100644 tests/helpers/generator-benchmark.ts

diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md
index f1a1880..dca3e1f 100644
--- a/docs/contributing/testing.md
+++ b/docs/contributing/testing.md
@@ -49,6 +49,28 @@ The default test suite must not require live network access. Tests should not co
 
 Network-facing code should be tested through injected or controlled boundaries whenever possible.
 
+## Performance Benchmarks
+
+`tests/generator-benchmark.test.ts` provides the bounded local benchmark harness for project generation. It measures all four bundled generation paths (`basic-app`, `full-stack`, `smart-contract`, `api-service`) through the same trusted template pipeline used by `stellarforge new`.
+
+The harness is deliberately scoped to template rendering only. It performs no dependency installation, no Git initialization, and no network access, so the reported timings reflect local generator cost rather than registry or install latency. Each timed iteration generates into an isolated temporary directory that is removed afterwards.
+
+The suite reports wall-clock statistics (mean, median, p95, min, max, standard deviation, and coefficient of variation) plus generated file counts and byte totals for each template. Report structure is deterministic: the environment metadata, iteration counts, template ordering, and artifact measurements are stable, so two runs on the same machine can be compared for regressions. Absolute timing is not asserted in CI because it varies by host.
+
+Run just the harness locally:
+
+```bash
+npx vitest run tests/generator-benchmark.test.ts
+```
+
+Deepen the run for a more stable baseline by raising the bounded iteration count through an environment variable:
+
+```bash
+STELLARFORGE_BENCHMARK_ITERATIONS=20 npx vitest run tests/generator-benchmark.test.ts
+```
+
+The iteration count is clamped to a safe maximum so the harness always stays bounded. Capture the printed report as the comparison baseline for later runs on the same platform.
+
 ## Running Tests
 
 From a clean checkout:
diff --git a/tests/generator-benchmark.test.ts b/tests/generator-benchmark.test.ts
new file mode 100644
index 0000000..1c3346a
--- /dev/null
+++ b/tests/generator-benchmark.test.ts
@@ -0,0 +1,167 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { generateProjectFromTemplate } from '../src/generator/template-generation.js';
+import {
+  TEMPLATE_IDS,
+  type TemplateId,
+} from '../src/generator/template-registry.js';
+import { resolveBundledTemplateRoot } from '../src/generator/template-sources.js';
+import {
+  createGeneratorBenchmarkFingerprint,
+  DEFAULT_BENCHMARK_ITERATIONS,
+  DEFAULT_BENCHMARK_WARMUP_ITERATIONS,
+  formatGeneratorBenchmarkText,
+  MAX_BENCHMARK_ITERATIONS,
+  resolveBenchmarkIterations,
+  runGeneratorBenchmark,
+  serializeGeneratorBenchmarkReport,
+  withTempDirectory,
+} from './helpers/index.js';
+
+const EXPECTED_MARKERS: Record<TemplateId, readonly string[]> = {
+  'basic-app': ['package.json', 'src/index.js', 'src/network.js'],
+  'full-stack': [
+    'package.json',
+    'backend/src/server.js',
+    'frontend/src/app.js',
+    'contracts/README.md',
+  ],
+  'smart-contract': [
+    'Cargo.toml',
+    join('contracts', 'hello', 'Cargo.toml'),
+    join('contracts', 'hello', 'src', 'lib.rs'),
+  ],
+  'api-service': [
+    'package.json',
+    'src/server.js',
+    'src/config.js',
+    'src/stellar-client.js',
+  ],
+};
+
+const NODE_TEMPLATES: readonly TemplateId[] = [
+  'basic-app',
+  'full-stack',
+  'api-service',
+];
+
+function generateTemplate(
+  root: string,
+  templateId: TemplateId,
+  projectName = 'benchmark-project',
+): string {
+  const destination = join(root, projectName);
+
+  generateProjectFromTemplate(
+    {
+      input: { projectName, cwd: root, templateId },
+      destination: { projectName, destination },
+      template: { templateId },
+    },
+    { templateRoot: resolveBundledTemplateRoot(templateId) },
+  );
+
+  return destination;
+}
+
+describe('generator performance benchmark', () => {
+  it('measures every bundled generation path offline and excludes dependency installs', () => {
+    const iterations = resolveBenchmarkIterations(
+      process.env['STELLARFORGE_BENCHMARK_ITERATIONS'],
+      DEFAULT_BENCHMARK_ITERATIONS,
+    );
+    const report = runGeneratorBenchmark({ iterations });
+
+    expect(report.templates.map((template) => template.templateId)).toEqual([
+      ...TEMPLATE_IDS,
+    ]);
+    expect(report.iterations).toBe(iterations);
+    expect(report.warmupIterations).toBe(DEFAULT_BENCHMARK_WARMUP_ITERATIONS);
+
+    for (const template of report.templates) {
+      expect(template.samplesMs).toHaveLength(iterations);
+
+      for (const sample of template.samplesMs) {
+        expect(Number.isFinite(sample)).toBe(true);
+        expect(sample).toBeGreaterThanOrEqual(0);
+      }
+
+      expect(template.generatedFileCount).toBeGreaterThan(0);
+      expect(template.generatedTotalBytes).toBeGreaterThan(0);
+      expect(template.producedNodeModules).toBe(false);
+
+      expect(template.timings.minMs).toBeLessThanOrEqual(
+        template.timings.medianMs,
+      );
+      expect(template.timings.medianMs).toBeLessThanOrEqual(
+        template.timings.maxMs,
+      );
+      expect(template.timings.p95Ms).toBeGreaterThanOrEqual(
+        template.timings.minMs,
+      );
+      expect(template.timings.p95Ms).toBeLessThanOrEqual(
+        template.timings.maxMs,
+      );
+      expect(template.timings.standardDeviationMs).toBeGreaterThanOrEqual(0);
+      expect(template.timings.coefficientOfVariation).toBeGreaterThanOrEqual(0);
+    }
+
+    console.log(formatGeneratorBenchmarkText(report));
+  });
+
+  it('generates the documented template artifacts without installing dependencies', async () => {
+    for (const templateId of TEMPLATE_IDS) {
+      await withTempDirectory((root) => {
+        const destination = generateTemplate(root, templateId);
+
+        for (const marker of EXPECTED_MARKERS[templateId]) {
+          expect(existsSync(join(destination, marker)), marker).toBe(true);
+        }
+
+        expect(existsSync(join(destination, 'node_modules'))).toBe(false);
+
+        if (NODE_TEMPLATES.includes(templateId)) {
+          const projectPackage = JSON.parse(
+            readFileSync(join(destination, 'package.json'), 'utf8'),
+          ) as { name?: unknown };
+
+          expect(projectPackage.name).toBe('benchmark-project');
+        }
+      });
+    }
+  });
+
+  it('produces structurally reproducible reports for regression comparison', () => {
+    const first = runGeneratorBenchmark({ iterations: 2 });
+    const second = runGeneratorBenchmark({ iterations: 2 });
+
+    expect(createGeneratorBenchmarkFingerprint(second)).toBe(
+      createGeneratorBenchmarkFingerprint(first),
+    );
+    expect(JSON.parse(serializeGeneratorBenchmarkReport(first))).toEqual(first);
+  });
+
+  it('defaults and bounds iteration overrides without unbounded runs', () => {
+    expect(resolveBenchmarkIterations(undefined, 3)).toBe(3);
+    expect(resolveBenchmarkIterations('', 3)).toBe(3);
+    expect(resolveBenchmarkIterations('5', 3)).toBe(5);
+    expect(resolveBenchmarkIterations('not-a-number', 3)).toBe(3);
+    expect(resolveBenchmarkIterations('-2', 3)).toBe(3);
+    expect(
+      resolveBenchmarkIterations(String(MAX_BENCHMARK_ITERATIONS + 10), 3),
+    ).toBe(MAX_BENCHMARK_ITERATIONS);
+
+    const report = runGeneratorBenchmark({
+      iterations: 1,
+      warmupIterations: 0,
+    });
+
+    expect(report.iterations).toBe(1);
+    expect(report.warmupIterations).toBe(0);
+
+    for (const template of report.templates) {
+      expect(template.samplesMs).toHaveLength(1);
+    }
+  });
+});
diff --git a/tests/helpers/generator-benchmark.ts b/tests/helpers/generator-benchmark.ts
new file mode 100644
index 0000000..74cf8d6
--- /dev/null
+++ b/tests/helpers/generator-benchmark.ts
@@ -0,0 +1,308 @@
+import { readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { generateProjectFromTemplate } from '../../src/generator/template-generation.js';
+import {
+  TEMPLATE_IDS,
+  type TemplateId,
+} from '../../src/generator/template-registry.js';
+import { resolveBundledTemplateRoot } from '../../src/generator/template-sources.js';
+import { createTempDirectoryFixture } from './temp-directory.js';
+
+export const DEFAULT_BENCHMARK_ITERATIONS = 3;
+export const DEFAULT_BENCHMARK_WARMUP_ITERATIONS = 2;
+export const MAX_BENCHMARK_ITERATIONS = 100;
+
+export interface TimingStatistics {
+  readonly minMs: number;
+  readonly maxMs: number;
+  readonly meanMs: number;
+  readonly medianMs: number;
+  readonly p95Ms: number;
+  readonly standardDeviationMs: number;
+  readonly coefficientOfVariation: number;
+}
+
+export interface TemplateBenchmarkResult {
+  readonly templateId: TemplateId;
+  readonly iterations: number;
+  readonly warmupIterations: number;
+  readonly samplesMs: readonly number[];
+  readonly timings: TimingStatistics;
+  readonly generatedFileCount: number;
+  readonly generatedTotalBytes: number;
+  readonly producedNodeModules: boolean;
+}
+
+export interface GeneratorBenchmarkEnvironment {
+  readonly nodeVersion: string;
+  readonly platform: NodeJS.Platform;
+  readonly arch: string;
+}
+
+export interface GeneratorBenchmarkReport {
+  readonly iterations: number;
+  readonly warmupIterations: number;
+  readonly environment: GeneratorBenchmarkEnvironment;
+  readonly templates: readonly TemplateBenchmarkResult[];
+}
+
+export interface GeneratorBenchmarkOptions {
+  readonly iterations?: number;
+  readonly warmupIterations?: number;
+  readonly templateIds?: readonly TemplateId[];
+}
+
+interface TreeMeasurement {
+  readonly fileCount: number;
+  readonly totalBytes: number;
+  readonly containsNodeModules: boolean;
+}
+
+function measureTree(root: string): TreeMeasurement {
+  let fileCount = 0;
+  let totalBytes = 0;
+  let containsNodeModules = false;
+
+  const walk = (directory: string): void => {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      if (entry.isDirectory()) {
+        if (entry.name === 'node_modules') {
+          containsNodeModules = true;
+          continue;
+        }
+
+        walk(join(directory, entry.name));
+        continue;
+      }
+
+      fileCount += 1;
+      totalBytes += statSync(join(directory, entry.name)).size;
+    }
+  };
+
+  walk(root);
+
+  return { fileCount, totalBytes, containsNodeModules };
+}
+
+function resolveBoundedCount(
+  requested: number | undefined,
+  fallback: number,
+): number {
+  if (
+    requested === undefined ||
+    !Number.isInteger(requested) ||
+    requested < 0
+  ) {
+    return fallback;
+  }
+
+  return Math.min(requested, MAX_BENCHMARK_ITERATIONS);
+}
+
+export function resolveBenchmarkIterations(
+  raw: string | undefined,
+  fallback: number,
+  maximum = MAX_BENCHMARK_ITERATIONS,
+): number {
+  if (raw === undefined || raw.trim() === '') {
+    return fallback;
+  }
+
+  const parsed = Number.parseInt(raw, 10);
+
+  if (!Number.isInteger(parsed) || parsed < 1) {
+    return fallback;
+  }
+
+  return Math.min(parsed, maximum);
+}
+
+function computeStatistics(samplesMs: readonly number[]): TimingStatistics {
+  const sorted = [...samplesMs].sort((a, b) => a - b);
+  const count = sorted.length;
+  const sum = sorted.reduce((total, value) => total + value, 0);
+  const meanMs = count === 0 ? 0 : sum / count;
+  const minMs = count === 0 ? 0 : (sorted[0] as number);
+  const maxMs = count === 0 ? 0 : (sorted[count - 1] as number);
+  const medianMs =
+    count === 0
+      ? 0
+      : count % 2 === 1
+        ? (sorted[(count - 1) / 2] as number)
+        : ((sorted[count / 2 - 1] as number) + (sorted[count / 2] as number)) /
+          2;
+  const p95Index = Math.max(0, Math.ceil(0.95 * count) - 1);
+  const p95Ms =
+    count === 0 ? 0 : (sorted[Math.min(p95Index, count - 1)] as number);
+  const variance =
+    count === 0
+      ? 0
+      : sorted.reduce((total, value) => total + (value - meanMs) ** 2, 0) /
+        count;
+  const standardDeviationMs = Math.sqrt(variance);
+  const coefficientOfVariation =
+    meanMs === 0 ? 0 : standardDeviationMs / meanMs;
+
+  return {
+    minMs,
+    maxMs,
+    meanMs,
+    medianMs,
+    p95Ms,
+    standardDeviationMs,
+    coefficientOfVariation,
+  };
+}
+
+function benchmarkTemplate(
+  templateId: TemplateId,
+  workRoot: string,
+  iterations: number,
+  warmupIterations: number,
+): TemplateBenchmarkResult {
+  const templateRoot = resolveBundledTemplateRoot(templateId);
+  const samplesMs: number[] = [];
+  let lastMeasurement: TreeMeasurement = {
+    fileCount: 0,
+    totalBytes: 0,
+    containsNodeModules: false,
+  };
+
+  const generate = (phase: string, index: number): TreeMeasurement => {
+    const projectName = `benchmark-${templateId}-${phase}${index}`;
+    const destination = join(workRoot, projectName);
+
+    const start = performance.now();
+    generateProjectFromTemplate(
+      {
+        input: { projectName, cwd: workRoot, templateId },
+        destination: { projectName, destination },
+        template: { templateId },
+      },
+      { templateRoot },
+    );
+    const elapsedMs = performance.now() - start;
+
+    samplesMs.push(elapsedMs);
+
+    return measureTree(destination);
+  };
+
+  for (let index = 0; index < warmupIterations; index += 1) {
+    lastMeasurement = generate('warmup-', index);
+  }
+
+  // Warmup output does not count toward the reported samples.
+  samplesMs.length = 0;
+
+  for (let index = 0; index < iterations; index += 1) {
+    lastMeasurement = generate('run-', index);
+  }
+
+  return {
+    templateId,
+    iterations,
+    warmupIterations,
+    samplesMs,
+    timings: computeStatistics(samplesMs),
+    generatedFileCount: lastMeasurement.fileCount,
+    generatedTotalBytes: lastMeasurement.totalBytes,
+    producedNodeModules: lastMeasurement.containsNodeModules,
+  };
+}
+
+export function runGeneratorBenchmark(
+  options: GeneratorBenchmarkOptions = {},
+): GeneratorBenchmarkReport {
+  const iterations = resolveBoundedCount(
+    options.iterations,
+    DEFAULT_BENCHMARK_ITERATIONS,
+  );
+  const warmupIterations = resolveBoundedCount(
+    options.warmupIterations,
+    DEFAULT_BENCHMARK_WARMUP_ITERATIONS,
+  );
+  const templateIds = options.templateIds ?? TEMPLATE_IDS;
+  const workRoot = createTempDirectoryFixture('stellarforge-benchmark-');
+
+  try {
+    const templates = templateIds.map((templateId) =>
+      benchmarkTemplate(
+        templateId,
+        workRoot.path,
+        iterations,
+        warmupIterations,
+      ),
+    );
+
+    return {
+      iterations,
+      warmupIterations,
+      environment: {
+        nodeVersion: process.version,
+        platform: process.platform,
+        arch: process.arch,
+      },
+      templates,
+    };
+  } finally {
+    workRoot.cleanup();
+  }
+}
+
+function formatMilliseconds(value: number): string {
+  return value.toFixed(3);
+}
+
+export function createGeneratorBenchmarkFingerprint(
+  report: GeneratorBenchmarkReport,
+): string {
+  return JSON.stringify({
+    iterations: report.iterations,
+    warmupIterations: report.warmupIterations,
+    templates: report.templates.map((template) => ({
+      templateId: template.templateId,
+      iterations: template.iterations,
+      warmupIterations: template.warmupIterations,
+      generatedFileCount: template.generatedFileCount,
+      generatedTotalBytes: template.generatedTotalBytes,
+      producedNodeModules: template.producedNodeModules,
+    })),
+  });
+}
+
+export function serializeGeneratorBenchmarkReport(
+  report: GeneratorBenchmarkReport,
+): string {
+  return JSON.stringify(report, null, 2);
+}
+
+export function formatGeneratorBenchmarkText(
+  report: GeneratorBenchmarkReport,
+): string {
+  const header = [
+    'Generator performance benchmark (offline template generation)',
+    `node=${report.environment.nodeVersion} platform=${report.environment.platform} arch=${report.environment.arch}`,
+    `iterations=${report.iterations} warmupIterations=${report.warmupIterations}`,
+    '',
+  ];
+
+  const rows = report.templates.map((template) =>
+    [
+      template.templateId.padEnd(16),
+      `mean=${formatMilliseconds(template.timings.meanMs)}ms`,
+      `median=${formatMilliseconds(template.timings.medianMs)}ms`,
+      `p95=${formatMilliseconds(template.timings.p95Ms)}ms`,
+      `min=${formatMilliseconds(template.timings.minMs)}ms`,
+      `max=${formatMilliseconds(template.timings.maxMs)}ms`,
+      `stddev=${formatMilliseconds(template.timings.standardDeviationMs)}ms`,
+      `cv=${template.timings.coefficientOfVariation.toFixed(3)}`,
+      `files=${template.generatedFileCount}`,
+      `bytes=${template.generatedTotalBytes}`,
+    ].join('  '),
+  );
+
+  return [...header, ...rows].join('\n');
+}
diff --git a/tests/helpers/index.ts b/tests/helpers/index.ts
index 32a9251..657268f 100644
--- a/tests/helpers/index.ts
+++ b/tests/helpers/index.ts
@@ -8,6 +8,21 @@ export {
   repositoryRoot,
   runBuiltCli,
 } from './cli-process.js';
+export {
+  createGeneratorBenchmarkFingerprint,
+  DEFAULT_BENCHMARK_ITERATIONS,
+  DEFAULT_BENCHMARK_WARMUP_ITERATIONS,
+  formatGeneratorBenchmarkText,
+  MAX_BENCHMARK_ITERATIONS,
+  resolveBenchmarkIterations,
+  runGeneratorBenchmark,
+  serializeGeneratorBenchmarkReport,
+  type GeneratorBenchmarkEnvironment,
+  type GeneratorBenchmarkOptions,
+  type GeneratorBenchmarkReport,
+  type TemplateBenchmarkResult,
+  type TimingStatistics,
+} from './generator-benchmark.js';
 export {
   createTempDirectoryFixture,
   withTempDirectory,