diff --git a/.changeset/quiet-dogs-smoke.md b/.changeset/quiet-dogs-smoke.md new file mode 100644 index 0000000..6d4ccab --- /dev/null +++ b/.changeset/quiet-dogs-smoke.md @@ -0,0 +1,5 @@ +--- +"@diginodes/stellarforge-cli": patch +--- + +Return the doctor diagnostic exit code when environment checks fail. \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47fcc75..d2c9cb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,9 @@ jobs: - name: Build run: npm run build + - name: E2E smoke + run: npm run test:e2e + platform: name: Platform / ${{ matrix.name }} strategy: diff --git a/docs/contributing/release-process.md b/docs/contributing/release-process.md index b2c71b3..bd37bb4 100644 --- a/docs/contributing/release-process.md +++ b/docs/contributing/release-process.md @@ -2,21 +2,23 @@ StellarForge CLI uses Semantic Versioning, Changesets, npm Trusted Publishing, and GitHub Releases. -## Safety state before the first supported publication +## Workflow and external controls -Release automation is intentionally merged in a non-publishing state. The workflow can create release PRs and validate/pack release artifacts, but the npm publish job runs only when **all** of these controls are in place: +Workflow code creates release PRs and validates/packs artifacts. Its publish job requests OIDC only in the `npm-release` environment and runs only when the repository variable `NPM_PUBLISH_ENABLED` is exactly `true`. + +The following prerequisites are configured administratively outside the repository workflow: 1. the `@diginodes/stellarforge-cli` npm scope/package is owned by the StellarForge maintainers through the `diginodes` npm organization; -2. npm Trusted Publishing is configured for: +2. npm Trusted Publishing is bound to: - GitHub organization: `DigiNodes` - repository: `stellarforge-cli` - workflow filename: `release.yml` - environment: `npm-release` - direct `npm publish` is allowed; 3. the GitHub `npm-release` environment exists and requires maintainer approval; -4. the repository variable `NPM_PUBLISH_ENABLED` is set to `true`. +4. a maintainer sets the repository variable `NPM_PUBLISH_ENABLED` to `true` for an approved release window. -The first three controls are configured. `NPM_PUBLISH_ENABLED` deliberately remains `false`, so release artifacts are built, tested, inspected, and packed while publication stops at the protected gate. +The first three prerequisites are configured. `NPM_PUBLISH_ENABLED` deliberately remains `false`, so release artifacts are built, tested, inspected, and packed while publication stops at the protected gate. The workflow does not create the npm Trusted Publisher, GitHub environment reviewers, or repository variable. No long-lived `NPM_TOKEN` is used. diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index f1a1880..1804f35 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -58,4 +58,6 @@ npm ci npm test ``` +`npm run test:e2e` builds and exercises the compiled CLI with temporary project directories and a fake Stellar executable; it does not contact live services. `npm run benchmark:doctor` runs five baseline samples and a timeout scenario with temporary fake PATH executables. It reports min/median/max timings without machine-dependent pass thresholds and requires a POSIX environment. + Before opening a pull request, run the full local validation set documented in the installation and contribution guides. diff --git a/package.json b/package.json index 1e6c1f2..294403b 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "changeset": "changeset", "changeset:status": "changeset status", "changeset:version": "changeset version", - "test:e2e": "node scripts/e2e-smoke.mjs", + "test:e2e": "npm run build && node scripts/e2e-smoke.mjs", + "benchmark:doctor": "npm run build && node scripts/doctor-benchmark.mjs", "release:dry-run": "npm pack --dry-run --json", "release:publish": "changeset publish" }, diff --git a/scripts/doctor-benchmark.mjs b/scripts/doctor-benchmark.mjs new file mode 100644 index 0000000..6316f82 --- /dev/null +++ b/scripts/doctor-benchmark.mjs @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { performance } from 'node:perf_hooks'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +assert.notEqual(process.platform, 'win32', 'This benchmark requires POSIX fake executables.'); + +const repositoryRoot = resolve(fileURLToPath(import.meta.url), '..', '..'); +const cliPath = join(repositoryRoot, 'dist', 'cli.js'); +const runCount = 5; +const root = mkdtempSync(join(tmpdir(), 'stellarforge-doctor-benchmark-')); +const fakeBin = join(root, 'bin'); +mkdirSync(fakeBin); +const responses = { + npm: [['--version', '11.5.1\n']], + git: [['--version', 'git version 2.49.0\n']], + rustc: [['--version', 'rustc 1.84.0 (benchmark)\n']], + cargo: [['--version', 'cargo 1.84.0 (benchmark)\n']], + stellar: [['version --only-version', '27.1.0\n']], + docker: [ + ['--version', 'Docker version 28.4.0, build benchmark\n'], + ['info --format {{.ServerVersion}}', '28.4.0\n'], + ], +}; + +function installFakeExecutables() { + for (const [name, commands] of Object.entries(responses)) { + const commandMap = JSON.stringify( + commands.map(([args, output]) => [JSON.stringify(args.split(' ')), output]), + ); + const timeoutBehavior = + name === 'rustc' + ? `if (process.env.STELLARFORGE_BENCHMARK_TIMEOUT === '1') { + writeFileSync(process.env.STELLARFORGE_BENCHMARK_STARTED, 'started'); + setTimeout(() => {}, 20_000); + process.stdin.resume(); +} else ` + : ''; + const script = `#!/usr/bin/env node +import { writeFileSync } from 'node:fs'; +const args = process.argv.slice(2); +${timeoutBehavior}{ + const output = new Map(${commandMap}).get(JSON.stringify(args)); + if (output === undefined) { + process.stderr.write('unexpected fake executable arguments\\n'); + process.exitCode = 64; + } else { + process.stdout.write(output); + } +} +`; + const executable = join(fakeBin, name); + writeFileSync(executable, script); + chmodSync(executable, 0o755); + } +} + +function runDoctor(timeoutMarker) { + const startedAt = performance.now(); + const result = spawnSync(process.execPath, [cliPath, 'doctor'], { + cwd: root, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ''}`, + STELLARFORGE_BENCHMARK_TIMEOUT: timeoutMarker ? '1' : '0', + STELLARFORGE_BENCHMARK_STARTED: join(root, 'timeout-started'), + }, + shell: false, + timeout: 30_000, + }); + + assert.equal(result.error, undefined, result.error?.message ?? 'doctor spawn failed'); + assert.equal(result.stderr, '', result.stderr); + return { + elapsedMs: Number((performance.now() - startedAt).toFixed(2)), + stdout: result.stdout, + status: result.status, + }; +} + +try { + installFakeExecutables(); + + const baselineSamples = Array.from({ length: runCount }, () => { + const result = runDoctor(false); + assert.equal(result.status, 0, result.stdout); + assert.match(result.stdout, /Overall health: ready\./); + return result.elapsedMs; + }).sort((left, right) => left - right); + + const timeoutResult = runDoctor(true); + assert.equal(timeoutResult.status, 5, timeoutResult.stdout); + assert.match(timeoutResult.stdout, /Rust is unavailable or could not be executed\./); + assert.equal(existsSync(join(root, 'timeout-started')), true); + + const timeoutSample = { + elapsedMs: timeoutResult.elapsedMs, + exitCode: timeoutResult.status, + timeoutObserved: true, + summary: timeoutResult.stdout.match(/Summary: .*\./)?.[0], + }; + + process.stdout.write( + `${JSON.stringify( + { + benchmark: 'doctor-runtime', + executableSource: 'temporary fake PATH executables; no network', + baseline: { + runs: runCount, + minMs: baselineSamples[0], + medianMs: baselineSamples[Math.floor(runCount / 2)], + maxMs: baselineSamples.at(-1), + }, + timeout: timeoutSample, + }, + null, + 2, + )}\n`, + ); +} finally { + rmSync(root, { recursive: true, force: true }); +} \ No newline at end of file diff --git a/scripts/e2e-smoke.mjs b/scripts/e2e-smoke.mjs index 511b31d..2c2bd99 100644 --- a/scripts/e2e-smoke.mjs +++ b/scripts/e2e-smoke.mjs @@ -13,11 +13,11 @@ import process from 'node:process'; import { delimiter, join } from 'node:path'; import { spawnSync } from 'node:child_process'; -const cliCommand = process.platform === 'win32' ? 'stellarforge.cmd' : 'stellarforge'; const repositoryRoot = process.cwd(); +const cliPath = join(repositoryRoot, 'dist', 'cli.js'); function runCli(args, cwd, env = process.env) { - return spawnSync(cliCommand, args, { + return spawnSync(process.execPath, [cliPath, ...args], { cwd, env, encoding: 'utf8', @@ -29,6 +29,7 @@ function runCli(args, cwd, env = process.env) { function expectSuccessful(result, label) { assert.equal(result.error, undefined, `${label}: ${result.error?.message ?? ''}`); assert.equal(result.status, 0, `${label}: ${result.stderr}`); + assert.equal(result.stderr, '', `${label} wrote unexpected stderr`); } const root = mkdtempSync(join(tmpdir(), 'stellarforge-e2e-')); @@ -47,9 +48,20 @@ try { const doctor = runCli(['doctor'], root); assert.equal(doctor.error, undefined, doctor.error?.message); - assert.ok([0, 5].includes(doctor.status)); - assert.match(doctor.stdout, /Summary:/); - assert.doesNotMatch(doctor.stdout, /SECRET_/); + assert.equal(doctor.stderr, ''); + const failedChecks = Number( + doctor.stdout.match(/Summary: \d+ passed, \d+ warnings, (\d+) failed\./)?.[1], + ); + assert.ok(Number.isInteger(failedChecks), 'doctor summary is missing'); + assert.equal(doctor.status, failedChecks > 0 ? 5 : 0); + for (const tool of ['Node.js', 'npm', 'Git', 'Rust', 'Cargo', 'Stellar CLI', 'Docker']) { + assert.ok(doctor.stdout.includes(`] ${tool}:`), `doctor omitted ${tool}`); + } + assert.match( + doctor.stdout, + /Overall health: (ready|usable with warnings|action required)\./, + ); + assert.doesNotMatch(doctor.stdout, /SECRET|TOKEN|PRIVATE KEY/i); const createBasic = runCli( ['new', 'smoke-basic', '--template', 'basic-app'], diff --git a/src/cli.ts b/src/cli.ts index b64c3ca..6bc91b3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,6 +1,10 @@ #!/usr/bin/env node import { runWithCliErrorBoundary } from './errors/index.js'; +import { EXIT_CODES } from './errors/errors.js'; import { runCli } from './main.js'; -process.exitCode = await runWithCliErrorBoundary(() => runCli()); +const boundaryExitCode = await runWithCliErrorBoundary(() => runCli()); +if (boundaryExitCode !== EXIT_CODES.success) { + process.exitCode = boundaryExitCode; +} diff --git a/tests/executable.test.ts b/tests/executable.test.ts index e621f4c..cb4868c 100644 --- a/tests/executable.test.ts +++ b/tests/executable.test.ts @@ -53,8 +53,12 @@ describe('StellarForge CLI executable', () => { 'runs default doctor checks safely in the CI toolchain', () => { const result = runBuiltCli(['doctor']); + const failedChecks = result.stdout.match( + /Summary: \d+ passed, \d+ warnings, (\d+) failed\./, + )?.[1]; - expect([0, 5]).toContain(result.status); + expect(failedChecks).toBeDefined(); + expect(result.status).toBe(Number(failedChecks) > 0 ? 5 : 0); expect(result.stderr).toBe(''); expect(result.stdout).toContain('[PASS] Node.js:'); expect(result.stdout).toContain('[PASS] npm:'); @@ -63,9 +67,6 @@ describe('StellarForge CLI executable', () => { expect(result.stdout).toContain('Cargo:'); expect(result.stdout).toContain('Stellar CLI:'); expect(result.stdout).toContain('Docker:'); - expect(result.stdout).toMatch( - /Summary: \d+ passed, \d+ warnings, \d+ failed\./, - ); }, DOCTOR_TEST_TIMEOUT_MS, ); diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index dd94afb..5466c6e 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -4,6 +4,32 @@ import { describe, expect, it } from 'vitest'; const root = join(import.meta.dirname, '..'); +function releaseJobBlock(workflow: string, name: string): string { + const lines = workflow.split('\n'); + const start = lines.indexOf(` ${name}:`); + if (start === -1) throw new Error(`Release workflow job not found: ${name}`); + + const nextJob = lines.findIndex( + (line, index) => index > start && /^ {2}[a-z][\w-]*:$/.test(line), + ); + return lines.slice(start, nextJob === -1 ? undefined : nextJob).join('\n'); +} + +function releaseJobPermissions(job: string): Record { + const lines = job.split('\n'); + const start = lines.findIndex((line) => line.startsWith(' permissions:')); + if (start === -1) throw new Error('Release workflow job has no permissions'); + if (lines[start]?.trim() === 'permissions: {}') return {}; + + const permissions: Record = {}; + for (const line of lines.slice(start + 1)) { + const entry = /^ {6}([\w-]+):\s*(\S+)\s*$/.exec(line); + if (!entry) break; + permissions[entry[1]!] = entry[2]!; + } + return permissions; +} + describe('release automation policy', () => { const workflow = readFileSync( join(root, '.github', 'workflows', 'release.yml'), @@ -41,10 +67,41 @@ describe('release automation policy', () => { expect(workflow).toContain( 'changesets/action/publish@ae32849d5ba541f9ae29e40e22a623bc13562f51', ); - expect(workflow.match(/id-token: write/g)).toHaveLength(1); expect(workflow).toContain('environment: npm-release'); }); + it('limits each job to its intended permissions and publish-only OIDC', () => { + const jobs = [ + 'select-mode', + 'version', + 'pack', + 'publish', + 'publish-disabled', + 'readiness', + ]; + const permissions = Object.fromEntries( + jobs.map((name) => [ + name, + releaseJobPermissions(releaseJobBlock(workflow, name)), + ]), + ); + + expect(permissions).toEqual({ + 'select-mode': { contents: 'read' }, + version: { contents: 'write', 'pull-requests': 'write' }, + pack: { contents: 'read' }, + publish: { contents: 'write', 'id-token': 'write' }, + 'publish-disabled': {}, + readiness: { contents: 'read' }, + }); + expect( + jobs.filter((name) => permissions[name]?.['id-token'] === 'write'), + ).toEqual(['publish']); + expect(releaseJobBlock(workflow, 'publish')).toContain( + ' environment: npm-release', + ); + }); + it('requires an explicit publication kill switch and contains no npm token secret', () => { expect(workflow).toContain("vars.NPM_PUBLISH_ENABLED == 'true'"); expect(workflow).not.toContain('NPM_TOKEN');