feat: add end-to-end testing and benchmark scripts; enhance release

.changeset/quiet-dogs-smoke.md

@@ -0,0 +1,5 @@
+---
+"@diginodes/stellarforge-cli": patch
+---
+
+Return the doctor diagnostic exit code when environment checks fail.

.github/workflows/ci.yml

@@ -49,6 +49,9 @@ jobs:
       - name: Build
         run: npm run build
 
+      - name: E2E smoke
+        run: npm run test:e2e
+
   platform:
     name: Platform / ${{ matrix.name }}
     strategy:

docs/contributing/release-process.md

@@ -2,21 +2,23 @@
 
 StellarForge CLI uses Semantic Versioning, Changesets, npm Trusted Publishing, and GitHub Releases.
 
-## Safety state before the first supported publication
+## Workflow and external controls
 
-Release automation is intentionally merged in a non-publishing state. The workflow can create release PRs and validate/pack release artifacts, but the npm publish job runs only when **all** of these controls are in place:
+Workflow code creates release PRs and validates/packs artifacts. Its publish job requests OIDC only in the `npm-release` environment and runs only when the repository variable `NPM_PUBLISH_ENABLED` is exactly `true`.
+
+The following prerequisites are configured administratively outside the repository workflow:
 
 1. the `@diginodes/stellarforge-cli` npm scope/package is owned by the StellarForge maintainers through the `diginodes` npm organization;
-2. npm Trusted Publishing is configured for:
+2. npm Trusted Publishing is bound to:
    - GitHub organization: `DigiNodes`
    - repository: `stellarforge-cli`
    - workflow filename: `release.yml`
    - environment: `npm-release`
    - direct `npm publish` is allowed;
 3. the GitHub `npm-release` environment exists and requires maintainer approval;
-4. the repository variable `NPM_PUBLISH_ENABLED` is set to `true`.
+4. a maintainer sets the repository variable `NPM_PUBLISH_ENABLED` to `true` for an approved release window.
 
-The first three controls are configured. `NPM_PUBLISH_ENABLED` deliberately remains `false`, so release artifacts are built, tested, inspected, and packed while publication stops at the protected gate.
+The first three prerequisites are configured. `NPM_PUBLISH_ENABLED` deliberately remains `false`, so release artifacts are built, tested, inspected, and packed while publication stops at the protected gate. The workflow does not create the npm Trusted Publisher, GitHub environment reviewers, or repository variable.
 
 No long-lived `NPM_TOKEN` is used.
 

docs/contributing/testing.md

@@ -58,4 +58,6 @@ npm ci
 npm test
 ```
 
+`npm run test:e2e` builds and exercises the compiled CLI with temporary project directories and a fake Stellar executable; it does not contact live services. `npm run benchmark:doctor` runs five baseline samples and a timeout scenario with temporary fake PATH executables. It reports min/median/max timings without machine-dependent pass thresholds and requires a POSIX environment.
+
 Before opening a pull request, run the full local validation set documented in the installation and contribution guides.

package.json

@@ -22,7 +22,8 @@
     "changeset": "changeset",
     "changeset:status": "changeset status",
     "changeset:version": "changeset version",
-    "test:e2e": "node scripts/e2e-smoke.mjs",
+    "test:e2e": "npm run build && node scripts/e2e-smoke.mjs",
+    "benchmark:doctor": "npm run build && node scripts/doctor-benchmark.mjs",
     "release:dry-run": "npm pack --dry-run --json",
     "release:publish": "changeset publish"
   },

scripts/doctor-benchmark.mjs

@@ -0,0 +1,134 @@
+import assert from 'node:assert/strict';
+import {
+  chmodSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { delimiter, join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { performance } from 'node:perf_hooks';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+assert.notEqual(process.platform, 'win32', 'This benchmark requires POSIX fake executables.');
+
+const repositoryRoot = resolve(fileURLToPath(import.meta.url), '..', '..');
+const cliPath = join(repositoryRoot, 'dist', 'cli.js');
+const runCount = 5;
+const root = mkdtempSync(join(tmpdir(), 'stellarforge-doctor-benchmark-'));
+const fakeBin = join(root, 'bin');
+mkdirSync(fakeBin);
+const responses = {
+  npm: [['--version', '11.5.1\n']],
+  git: [['--version', 'git version 2.49.0\n']],
+  rustc: [['--version', 'rustc 1.84.0 (benchmark)\n']],
+  cargo: [['--version', 'cargo 1.84.0 (benchmark)\n']],
+  stellar: [['version --only-version', '27.1.0\n']],
+  docker: [
+    ['--version', 'Docker version 28.4.0, build benchmark\n'],
+    ['info --format {{.ServerVersion}}', '28.4.0\n'],
+  ],
+};
+
+function installFakeExecutables() {
+  for (const [name, commands] of Object.entries(responses)) {
+    const commandMap = JSON.stringify(
+      commands.map(([args, output]) => [JSON.stringify(args.split(' ')), output]),
+    );
+    const timeoutBehavior =
+      name === 'rustc'
+        ? `if (process.env.STELLARFORGE_BENCHMARK_TIMEOUT === '1') {
+  writeFileSync(process.env.STELLARFORGE_BENCHMARK_STARTED, 'started');
+  setTimeout(() => {}, 20_000);
+  process.stdin.resume();
+} else `
+        : '';
+    const script = `#!/usr/bin/env node
+import { writeFileSync } from 'node:fs';
+const args = process.argv.slice(2);
+${timeoutBehavior}{
+  const output = new Map(${commandMap}).get(JSON.stringify(args));
+  if (output === undefined) {
+    process.stderr.write('unexpected fake executable arguments\\n');
+    process.exitCode = 64;
+  } else {
+    process.stdout.write(output);
+  }
+}
+`;
+    const executable = join(fakeBin, name);
+    writeFileSync(executable, script);
+    chmodSync(executable, 0o755);
+  }
+}
+
+function runDoctor(timeoutMarker) {
+  const startedAt = performance.now();
+  const result = spawnSync(process.execPath, [cliPath, 'doctor'], {
+    cwd: root,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`,
+      STELLARFORGE_BENCHMARK_TIMEOUT: timeoutMarker ? '1' : '0',
+      STELLARFORGE_BENCHMARK_STARTED: join(root, 'timeout-started'),
+    },
+    shell: false,
+    timeout: 30_000,
+  });
+
+  assert.equal(result.error, undefined, result.error?.message ?? 'doctor spawn failed');
+  assert.equal(result.stderr, '', result.stderr);
+  return {
+    elapsedMs: Number((performance.now() - startedAt).toFixed(2)),
+    stdout: result.stdout,
+    status: result.status,
+  };
+}
+
+try {
+  installFakeExecutables();
+
+  const baselineSamples = Array.from({ length: runCount }, () => {
+    const result = runDoctor(false);
+    assert.equal(result.status, 0, result.stdout);
+    assert.match(result.stdout, /Overall health: ready\./);
+    return result.elapsedMs;
+  }).sort((left, right) => left - right);
+
+  const timeoutResult = runDoctor(true);
+  assert.equal(timeoutResult.status, 5, timeoutResult.stdout);
+  assert.match(timeoutResult.stdout, /Rust is unavailable or could not be executed\./);
+  assert.equal(existsSync(join(root, 'timeout-started')), true);
+
+  const timeoutSample = {
+    elapsedMs: timeoutResult.elapsedMs,
+    exitCode: timeoutResult.status,
+    timeoutObserved: true,
+    summary: timeoutResult.stdout.match(/Summary: .*\./)?.[0],
+  };
+
+  process.stdout.write(
+    `${JSON.stringify(
+      {
+        benchmark: 'doctor-runtime',
+        executableSource: 'temporary fake PATH executables; no network',
+        baseline: {
+          runs: runCount,
+          minMs: baselineSamples[0],
+          medianMs: baselineSamples[Math.floor(runCount / 2)],
+          maxMs: baselineSamples.at(-1),
+        },
+        timeout: timeoutSample,
+      },
+      null,
+      2,
+    )}\n`,
+  );
+} finally {
+  rmSync(root, { recursive: true, force: true });
+}

scripts/e2e-smoke.mjs

@@ -13,11 +13,11 @@ import process from 'node:process';
 import { delimiter, join } from 'node:path';
 import { spawnSync } from 'node:child_process';
 
-const cliCommand = process.platform === 'win32' ? 'stellarforge.cmd' : 'stellarforge';
 const repositoryRoot = process.cwd();
+const cliPath = join(repositoryRoot, 'dist', 'cli.js');
 
 function runCli(args, cwd, env = process.env) {
-  return spawnSync(cliCommand, args, {
+  return spawnSync(process.execPath, [cliPath, ...args], {
     cwd,
     env,
     encoding: 'utf8',
@@ -29,6 +29,7 @@ function runCli(args, cwd, env = process.env) {
 function expectSuccessful(result, label) {
   assert.equal(result.error, undefined, `${label}: ${result.error?.message ?? ''}`);
   assert.equal(result.status, 0, `${label}: ${result.stderr}`);
+  assert.equal(result.stderr, '', `${label} wrote unexpected stderr`);
 }
 
 const root = mkdtempSync(join(tmpdir(), 'stellarforge-e2e-'));
@@ -47,9 +48,20 @@ try {
 
   const doctor = runCli(['doctor'], root);
   assert.equal(doctor.error, undefined, doctor.error?.message);
-  assert.ok([0, 5].includes(doctor.status));
-  assert.match(doctor.stdout, /Summary:/);
-  assert.doesNotMatch(doctor.stdout, /SECRET_/);
+  assert.equal(doctor.stderr, '');
+  const failedChecks = Number(
+    doctor.stdout.match(/Summary: \d+ passed, \d+ warnings, (\d+) failed\./)?.[1],
+  );
+  assert.ok(Number.isInteger(failedChecks), 'doctor summary is missing');
+  assert.equal(doctor.status, failedChecks > 0 ? 5 : 0);
+  for (const tool of ['Node.js', 'npm', 'Git', 'Rust', 'Cargo', 'Stellar CLI', 'Docker']) {
+    assert.ok(doctor.stdout.includes(`] ${tool}:`), `doctor omitted ${tool}`);
+  }
+  assert.match(
+    doctor.stdout,
+    /Overall health: (ready|usable with warnings|action required)\./,
+  );
+  assert.doesNotMatch(doctor.stdout, /SECRET|TOKEN|PRIVATE KEY/i);
 
   const createBasic = runCli(
     ['new', 'smoke-basic', '--template', 'basic-app'],

src/cli.ts

@@ -1,6 +1,10 @@
 #!/usr/bin/env node
 
 import { runWithCliErrorBoundary } from './errors/index.js';
+import { EXIT_CODES } from './errors/errors.js';
 import { runCli } from './main.js';
 
-process.exitCode = await runWithCliErrorBoundary(() => runCli());
+const boundaryExitCode = await runWithCliErrorBoundary(() => runCli());
+if (boundaryExitCode !== EXIT_CODES.success) {
+  process.exitCode = boundaryExitCode;
+}

tests/executable.test.ts

@@ -53,8 +53,12 @@ describe('StellarForge CLI executable', () => {
     'runs default doctor checks safely in the CI toolchain',
     () => {
       const result = runBuiltCli(['doctor']);
+      const failedChecks = result.stdout.match(
+        /Summary: \d+ passed, \d+ warnings, (\d+) failed\./,
+      )?.[1];
 
-      expect([0, 5]).toContain(result.status);
+      expect(failedChecks).toBeDefined();
+      expect(result.status).toBe(Number(failedChecks) > 0 ? 5 : 0);
       expect(result.stderr).toBe('');
       expect(result.stdout).toContain('[PASS] Node.js:');
       expect(result.stdout).toContain('[PASS] npm:');
@@ -63,9 +67,6 @@ describe('StellarForge CLI executable', () => {
       expect(result.stdout).toContain('Cargo:');
       expect(result.stdout).toContain('Stellar CLI:');
       expect(result.stdout).toContain('Docker:');
-      expect(result.stdout).toMatch(
-        /Summary: \d+ passed, \d+ warnings, \d+ failed\./,
-      );
     },
     DOCTOR_TEST_TIMEOUT_MS,
   );

tests/release-workflow.test.ts

@@ -4,6 +4,32 @@ import { describe, expect, it } from 'vitest';
 
 const root = join(import.meta.dirname, '..');
 
+function releaseJobBlock(workflow: string, name: string): string {
+  const lines = workflow.split('\n');
+  const start = lines.indexOf(`  ${name}:`);
+  if (start === -1) throw new Error(`Release workflow job not found: ${name}`);
+
+  const nextJob = lines.findIndex(
+    (line, index) => index > start && /^ {2}[a-z][\w-]*:$/.test(line),
+  );
+  return lines.slice(start, nextJob === -1 ? undefined : nextJob).join('\n');
+}
+
+function releaseJobPermissions(job: string): Record<string, string> {
+  const lines = job.split('\n');
+  const start = lines.findIndex((line) => line.startsWith('    permissions:'));
+  if (start === -1) throw new Error('Release workflow job has no permissions');
+  if (lines[start]?.trim() === 'permissions: {}') return {};
+
+  const permissions: Record<string, string> = {};
+  for (const line of lines.slice(start + 1)) {
+    const entry = /^ {6}([\w-]+):\s*(\S+)\s*$/.exec(line);
+    if (!entry) break;
+    permissions[entry[1]!] = entry[2]!;
+  }
+  return permissions;
+}
+
 describe('release automation policy', () => {
   const workflow = readFileSync(
     join(root, '.github', 'workflows', 'release.yml'),
@@ -41,10 +67,41 @@ describe('release automation policy', () => {
     expect(workflow).toContain(
       'changesets/action/publish@ae32849d5ba541f9ae29e40e22a623bc13562f51',
     );
-    expect(workflow.match(/id-token: write/g)).toHaveLength(1);
     expect(workflow).toContain('environment: npm-release');
   });
 
+  it('limits each job to its intended permissions and publish-only OIDC', () => {
+    const jobs = [
+      'select-mode',
+      'version',
+      'pack',
+      'publish',
+      'publish-disabled',
+      'readiness',
+    ];
+    const permissions = Object.fromEntries(
+      jobs.map((name) => [
+        name,
+        releaseJobPermissions(releaseJobBlock(workflow, name)),
+      ]),
+    );
+
+    expect(permissions).toEqual({
+      'select-mode': { contents: 'read' },
+      version: { contents: 'write', 'pull-requests': 'write' },
+      pack: { contents: 'read' },
+      publish: { contents: 'write', 'id-token': 'write' },
+      'publish-disabled': {},
+      readiness: { contents: 'read' },
+    });
+    expect(
+      jobs.filter((name) => permissions[name]?.['id-token'] === 'write'),
+    ).toEqual(['publish']);
+    expect(releaseJobBlock(workflow, 'publish')).toContain(
+      '    environment: npm-release',
+    );
+  });
+
   it('requires an explicit publication kill switch and contains no npm token secret', () => {
     expect(workflow).toContain("vars.NPM_PUBLISH_ENABLED == 'true'");
     expect(workflow).not.toContain('NPM_TOKEN');