fix(ci): run Web tests and preserve audit evidence (successor to #169)

.github/workflows/ci.yml

@@ -251,7 +251,24 @@ jobs:
 
       - name: Audit dependencies
         working-directory: web
-        run: npm run audit
+        # shell: bash is load-bearing: the default bash -e {0} shell has no
+        # pipefail, so tee would report its own exit status and a failing audit
+        # would pass this gate.
+        shell: bash
+        run: npm run audit 2>&1 | tee "${RUNNER_TEMP}/web-audit-transcript.txt"
+
+      - name: Upload audit evidence
+        if: always() && !cancelled()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: web-audit-transcript-${{ github.sha }}
+          path: ${{ runner.temp }}/web-audit-transcript.txt
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Run unit tests
+        working-directory: web
+        run: npm test
 
       - name: Lint
         working-directory: web
@@ -351,6 +368,80 @@ jobs:
           node --check platforms.js
           npm pack --dry-run --ignore-scripts
 
+  windows-audit-evidence:
+    name: Windows audit evidence
+    # The batch helper exists to prove audit-retry.mjs works on a real cmd.exe,
+    # where only `npm run` supplies npm_execpath. Its regression test replays the
+    # batch source against a stub npm.cmd, so it proves the mechanics but never
+    # the execution. This job is the execution, and it is a job rather than an
+    # `if: runner.os == 'Windows'` step inside npm-wrapper-compatibility so the
+    # check name itself documents the Windows dependency.
+    runs-on: windows-2025
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: "24"
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+
+      - name: Install dependencies
+        working-directory: web
+        run: npm ci
+
+      - name: Test Windows audit evidence helper
+        run: node --test scripts/test_win_audit_verify.mjs
+
+      - name: Run Windows audit evidence helper against the real registry
+        working-directory: web
+        # shell: bash is load-bearing: with the default shell there is no
+        # pipefail, so tee's exit status would replace cmd.exe's and the
+        # helper's nonzero-audit assertion would become unfalsifiable.
+        shell: bash
+        run: |
+          # //d //c, not /d /c: git-bash rewrites a leading /d and /c into D:/
+          # and C:/, which leaves cmd.exe with no option flags. It then prints
+          # its interactive banner and exits 0, so the helper never runs and the
+          # job goes green on an empty transcript.
+          cmd.exe //d //c "..\scripts\win-audit-verify.bat" 2>&1 \
+            | tee "${RUNNER_TEMP}/win-audit-transcript.txt"
+
+      - name: Verify the transcript is real evidence
+        shell: bash
+        # A successful cmd.exe proves nothing on its own; the transcript has to
+        # show the helper actually reached its report and gated on a passing
+        # audit, or this job would silently certify nothing.
+        run: |
+          transcript="${RUNNER_TEMP}/win-audit-transcript.txt"
+          for needle in \
+            "\[win-audit-verify\] starting npm run audit" \
+            "\[win-audit-verify\] finished at" \
+            "\[win-audit-verify\] npm run audit exit code: 0" \
+            "found 0 vulnerabilities"
+          do
+            if ! grep -q -- "$needle" "$transcript"; then
+              echo "::error::win-audit-verify.bat did not produce '$needle'; the Windows audit evidence is not real."
+              sed -n '1,40p' "$transcript"
+              exit 1
+            fi
+          done
+
+      - name: Upload Windows audit evidence
+        if: always() && !cancelled()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: win-audit-transcript-${{ github.sha }}
+          path: ${{ runner.temp }}/win-audit-transcript.txt
+          if-no-files-found: error
+          retention-days: 14
+
   deployment:
     name: Deployment profiles
     runs-on: ubuntu-24.04

CHANGELOG.md

@@ -96,6 +96,19 @@ The project publishes 0.x prerelease versions; a stable release line is not yet
   differs. `deploy/compose/compose.yaml` previously carried the same broken
   reference, so the documented self-hosted path would have failed on a cold
   host even though no workflow exercises it.
+- CI Web job now runs unit tests (`npm test`), including audit retry regression
+  tests. `npm run audit` retries recognized transient registry failures up to
+  three attempts per threshold (two retries), with a 60-second limit per attempt
+  and portable backoff. It starts npm through Node on Windows, fails immediately
+  for vulnerabilities, unknown errors or incomplete runs, and echoes the output
+  of every attempt it retries so a self-healing failure still leaves a trace.
+  That transcript is uploaded as a downloadable artifact
+  (`web-audit-transcript-<sha>`, 14-day retention) on the success and failure
+  paths alike, and because the step merges stderr into stdout the artifact also
+  carries the per-attempt diagnostics. A dedicated `Windows audit evidence` job
+  runs the audit through `scripts/win-audit-verify.bat` on a real Windows runner
+  and uploads `win-audit-transcript-<sha>`, so the helper is executed rather
+  than only replayed against a stub by its fixture test.
 - A configured URL that carries credentials in a shape `url.Parse` does not
   report as userinfo no longer reaches output. `admin:pw@host` parses as
   `Scheme="admin"` with the credential in `Opaque` and `User` unset, so an

scripts/test_win_audit_verify.mjs

@@ -0,0 +1,55 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+
+assert.equal(process.platform, "win32", "Run this process regression in the Windows CI job");
+const source = readFileSync(new URL("./win-audit-verify.bat", import.meta.url), "utf8");
+
+function invoke(script, status) {
+  const directory = mkdtempSync(join(tmpdir(), "mem audit evidence "));
+  try {
+    writeFileSync(join(directory, "verify.bat"), script);
+    // cmd.exe resolves the real .cmd fixture from its working directory.
+    // No registry, installed package, or user npm configuration is involved.
+    writeFileSync(join(directory, "npm.cmd"),
+      `@echo off\r\necho fixture npm audit status: ${status}\r\nexit /b ${status}\r\n`);
+    const result = spawnSync(process.env.ComSpec || "cmd.exe", ["/d", "/c", "verify.bat"], {
+      cwd: directory,
+      encoding: "utf8",
+      timeout: 15_000,
+      env: { ...process.env, AUDIT_RC: "" },
+    });
+    assert.ifError(result.error);
+    assert.match(result.stdout, new RegExp(`fixture npm audit status: ${status}`));
+    return result;
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+}
+
+for (const status of [0, 7]) {
+  test(`prints completed evidence and preserves audit exit ${status}`, () => {
+    const result = invoke(source, status);
+    assert.equal(result.status, status);
+    assert.match(result.stdout, /\[win-audit-verify\] finished at/);
+    assert.match(result.stdout, new RegExp(`npm run audit exit code: ${status}`));
+  });
+}
+
+test("negative control: omitting CALL loses the post-audit evidence", () => {
+  const broken = source.replace("call npm run audit", "npm run audit");
+  assert.notEqual(broken, source);
+  const result = invoke(broken, 7);
+  assert.doesNotMatch(result.stdout, /\[win-audit-verify\] finished at/);
+});
+
+test("negative control: separate ENDLOCAL loses a nonzero saved exit status", () => {
+  const broken = source.replace("endlocal & exit /b %AUDIT_RC%", "endlocal\r\nexit /b %AUDIT_RC%");
+  assert.notEqual(broken, source);
+  const result = invoke(broken, 7);
+  assert.match(result.stdout, /npm run audit exit code: 7/);
+  assert.equal(result.status, 0);
+});

scripts/win-audit-verify.bat

@@ -0,0 +1,41 @@
+@echo off
+REM win-audit-verify.bat — Verify audit-retry.mjs works on real Windows.
+REM Must be run from a real Windows terminal (cmd.exe), NOT WSL.
+REM Captures: commit, npm_execpath, full audit output, exit code.
+
+setlocal enabledelayedexpansion
+
+echo === win-audit-verify ===
+echo.
+
+REM 1. Show which commit is being tested
+echo --- git head ---
+git rev-parse HEAD
+git log -1 --format=^"%%h %%s^"
+echo.
+
+REM 2. Show Node version
+echo --- node ---
+node --version
+echo.
+
+REM 3. Prove npm_execpath is set by npm run, and that a direct node.exe
+REM    invocation does NOT have it (the failure mode from the review).
+echo --- npm_execpath probe via node -e (should be EMPTY) ---
+node -e "console.log('npm_execpath=' + (process.env.npm_execpath || '(unset)'))"
+echo.
+
+REM 4. Run the actual audit via npm run — this is the path that supplies npm_execpath.
+echo --- npm run audit (full output) ---
+echo [win-audit-verify] starting npm run audit at %DATE% %TIME%
+call npm run audit
+set AUDIT_RC=!ERRORLEVEL!
+echo [win-audit-verify] finished at %DATE% %TIME%
+echo.
+
+REM 5. Report exit code
+echo --- result ---
+echo [win-audit-verify] npm run audit exit code: !AUDIT_RC!
+echo.
+
+endlocal & exit /b %AUDIT_RC%

web/audit-retry.mjs

@@ -0,0 +1,138 @@
+import { spawnSync } from "node:child_process";
+import { setTimeout as sleep } from "node:timers/promises";
+import { pathToFileURL } from "node:url";
+
+const MAX_ATTEMPTS = 3;
+const BACKOFF_MS = 10_000;
+// At most six 60-second attempts and four 10-second backoffs across both thresholds.
+const ATTEMPT_TIMEOUT_MS = 60_000;
+
+const NETWORK_PATTERNS = [
+  "network timeout",
+  "503 Service Unavailable",
+  "ECONNRESET",
+  "ETIMEDOUT",
+];
+
+const VULNERABILITY_PATTERNS = [
+  "found \\d+ vulnerabilit(?:y|ies)",
+  "npm audit report",
+  "vulnerabilities found",
+];
+
+const COMMANDS = [
+  {
+    label: "production dependencies (moderate threshold)",
+    args: ["audit", "--omit=dev", "--audit-level=moderate", "--fetch-timeout=45000"],
+  },
+  {
+    label: "all dependencies (high threshold)",
+    args: ["audit", "--audit-level=high", "--fetch-timeout=45000"],
+  },
+];
+
+function isNetworkError(stderr) {
+  return NETWORK_PATTERNS.some((p) => stderr.toLowerCase().includes(p.toLowerCase()));
+}
+
+function isVulnerabilityReport(stdout) {
+  return VULNERABILITY_PATTERNS.some((p) => new RegExp(p, "i").test(stdout));
+}
+
+async function runWithRetry(label, args, { spawn, wait, npmExecPath, stdout, stderr }) {
+  let result;
+  const finish = () => {
+    stdout.write(result.stdout ?? "");
+    stderr.write(result.stderr ?? "");
+    if (result.error) {
+      stderr.write(`[audit-retry] ${result.error.code ?? "spawn error"}: ${result.error.message}\n`);
+    }
+    if (result.signal) {
+      stderr.write(`[audit-retry] terminated by ${result.signal}.\n`);
+    }
+    return Number.isInteger(result.status) && result.status > 0 && result.status <= 255 ? result.status : 1;
+  };
+
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    const ts = new Date().toISOString();
+    stderr.write(
+      `[audit-retry] ${ts} — ${label} (attempt ${attempt}/${MAX_ATTEMPTS})\n`
+    );
+
+    // npm run supplies the CLI path, including on Windows where npm is a .cmd
+    // shim that cannot be launched directly with shell-free spawnSync.
+    result = spawn(npmExecPath ? process.execPath : "npm", npmExecPath ? [npmExecPath, ...args] : args, {
+      encoding: "utf8",
+      stdio: ["inherit", "pipe", "pipe"],
+      timeout: ATTEMPT_TIMEOUT_MS,
+      killSignal: "SIGKILL",
+    });
+
+    // An interrupted or unstarted audit is never a valid audit result, even
+    // when its partial output happens to mention a transient network error.
+    if (result.error || result.signal || !Number.isInteger(result.status) || result.status < 0 || result.status > 255) {
+      stderr.write(`[audit-retry] ${label} did not complete — not retrying.\n`);
+      return finish();
+    }
+
+    if (result.status === 0) {
+      stdout.write(result.stdout ?? "");
+      stderr.write(result.stderr ?? "");
+      stderr.write(`[audit-retry] ${label} passed.\n`);
+      return 0;
+    }
+
+    const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
+
+    if (isVulnerabilityReport(output)) {
+      stderr.write(
+        `[audit-retry] ${label} found real vulnerabilities — not retrying.\n`
+      );
+      return finish();
+    }
+
+    if (isNetworkError(output)) {
+      if (attempt < MAX_ATTEMPTS) {
+        stderr.write(
+          `[audit-retry] ${label} hit a network error — will retry.\n` +
+          // An attempt that self-heals would otherwise leave no trace, and
+          // stdout stays reserved for the final audit report.
+          `[audit-retry] ${label} attempt ${attempt}/${MAX_ATTEMPTS} output:\n${output.trimEnd()}\n`
+        );
+        await wait(BACKOFF_MS);
+        continue;
+      }
+      stderr.write(`[audit-retry] ${label} exhausted ${MAX_ATTEMPTS} attempts.\n`);
+      return finish();
+    }
+
+    stderr.write(
+      `[audit-retry] ${label} failed with unrecognized error — not retrying.\n`
+    );
+    return finish();
+  }
+}
+
+export async function runAudits({
+  spawn = spawnSync,
+  wait = sleep,
+  npmExecPath = process.env.npm_execpath,
+  platform = process.platform,
+  stdout = process.stdout,
+  stderr = process.stderr,
+} = {}) {
+  if (platform === "win32" && !npmExecPath) {
+    stderr.write("[audit-retry] Run npm run audit so npm supplies its CLI path on Windows.\n");
+    return 1;
+  }
+
+  for (const cmd of COMMANDS) {
+    const code = await runWithRetry(cmd.label, cmd.args, { spawn, wait, npmExecPath, stdout, stderr });
+    if (code !== 0) return code;
+  }
+  return 0;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  process.exitCode = await runAudits();
+}