feat(test): run local suites in capped disposable containers

cli/__tests__/container-test-runner.test.mjs

@@ -0,0 +1,131 @@
+import { describe, test, expect } from '@jest/globals';
+import { mkdtemp, readFile, writeFile, chmod, mkdir, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { acquireLock, ciImages, selectSuite, validatePolicy, limits } from '../../scripts/container-tests/run.mjs';
+
+const checkout = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
+const sample = (lockPath) => ({ lockPath, nodeMemory: '2g', serviceMemory: '1g', cpus: 1, serviceCpus: 1,
+  workTmpfs: '1g', tempTmpfs: '128m', homeTmpfs: '128m', serviceTmpfs: '256m', logMaxSize: '1m', logMaxFiles: 1 });
+const usingTemp = async (fn) => {
+  const dir = await mkdtemp(resolve(tmpdir(), 'container-runner-test-'));
+  try { await fn(dir); } finally { await rm(dir, { recursive: true, force: true }); }
+};
+const fakeTools = async (dir) => {
+  const bin = resolve(dir, 'bin'); await mkdir(bin);
+  for (const name of ['docker', 'git']) {
+    const path = resolve(bin, name);
+    await writeFile(path, `#!/usr/bin/env node
+const fs = require('fs');
+const args = process.argv.slice(2);
+if (${JSON.stringify(name)} === 'git') { process.stdout.write('package.json\\0'); process.exit(0); }
+fs.appendFileSync(process.env.TRACE, JSON.stringify(args)+'\\n');
+if (args[0] === 'pull' && process.env.FAIL_PULL) process.exit(17);
+if (args[0] === 'image') process.stdout.write('sha256:'+args.at(-1).split(':')[0]);
+if (args[0] === 'inspect') process.stdout.write('healthy');
+if (args[0] === 'rm' && process.env.FAIL_CLEANUP) process.exit(1);
+if (args[0] === 'ps') process.stdout.write('container-id');
+if (args[0] === 'run' && args.includes('bash')) {
+  if (process.env.SIGNAL_PARENT) { process.kill(process.ppid, process.env.SIGNAL_PARENT); setTimeout(() => process.exit(0), 100); }
+  else process.exit(Number(process.env.SUITE_EXIT || 0));
+}
+`);
+    await chmod(path, 0o700);
+  }
+  return bin;
+};
+const invoke = async (dir, suite, extra = {}) => {
+  const policyFile = resolve(dir, 'policy.json');
+  await writeFile(policyFile, JSON.stringify(sample(resolve(dir, 'lock'))));
+  const bin = await fakeTools(dir);
+  const result = spawnSync(process.execPath, [resolve(checkout, 'scripts/container-tests/run.mjs'), suite], {
+    env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, TRACE: resolve(dir, 'trace'), COMMONLY_TEST_CONTAINER_POLICY: policyFile, ...extra },
+    encoding: 'utf8', timeout: 15000,
+  });
+  const trace = (await readFile(resolve(dir, 'trace'), 'utf8').catch(() => '')).trim().split('\n').filter(Boolean).map(JSON.parse);
+  return { result, trace };
+};
+
+describe('container test runner boundaries', () => {
+  test('uses CI image tags and refuses conflicting versions', async () => {
+    const workflow = await readFile(resolve(checkout, '.github/workflows/tests.yml'), 'utf8');
+    expect(ciImages(workflow)).toEqual({ node: 'node:22', postgres: 'postgres:16', mongo: 'mongo:7' });
+    expect(() => ciImages(`${workflow}\nnode-version: '24'`)).toThrow('disagree');
+  });
+  test('requires every SRE limit before execution', () => {
+    expect(() => validatePolicy({})).toThrow('lockPath');
+    for (const key of Object.keys(sample('/tmp/test-lock'))) {
+      const policy = sample('/tmp/test-lock'); delete policy[key];
+      expect(() => validatePolicy(policy)).toThrow();
+    }
+    expect(limits(validatePolicy(sample('/tmp/test-lock')), '2g')).toEqual(expect.arrayContaining([
+      '--read-only', '--memory', '2g', '--memory-swap', '--cpus', '1', '--log-driver', 'json-file', 'max-size=1m', 'max-file=1',
+    ]));
+  });
+  test('selects exact Jest files and refuses traversal or arbitrary commands', () => {
+    expect(selectSuite(['backend/__tests__/service/foo.test.js'])).toEqual({ suite: 'backend', integration: true, path: '__tests__/service/foo.test.js' });
+    for (const input of ['../../oops', 'cli/../backend/a.test.js', '--privileged', 'backend;echo hacked', '/tmp/a.test.js']) {
+      expect(() => selectSuite([input])).toThrow();
+    }
+    expect(() => selectSuite(['cli', '--watch'])).toThrow();
+  });
+  test('lock contention waits for the active owner, then acquires', async () => usingTemp(async (dir) => {
+    const path = resolve(dir, 'lock'); const release = await acquireLock(path);
+    let waits = 0;
+    const nextRelease = await acquireLock(path, { wait: async () => { waits += 1; await release(); } });
+    expect(waits).toBe(1);
+    expect(Number(await readFile(resolve(path, 'pid'), 'utf8'))).toBe(process.pid);
+    await nextRelease();
+  }));
+  test('dead PID is the only automatic stale-lock recovery', async () => usingTemp(async (dir) => {
+    const path = resolve(dir, 'lock');
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      await mkdir(path); await writeFile(resolve(path, 'pid'), '2147483647');
+      const release = await acquireLock(path, { alive: () => false, onStale: async (pid) => {
+        expect(pid).toBe(2147483647);
+        // Cleanup must finish while the old lock still excludes other runs.
+        expect(await readFile(resolve(path, 'pid'), 'utf8')).toBe(String(pid));
+        expect((await stat(`${path}.reap`)).isDirectory()).toBe(true);
+      }, wait: async () => { throw new Error('recovery mutex leaked'); } });
+      await expect(stat(`${path}.reap`)).rejects.toMatchObject({ code: 'ENOENT' });
+      expect(Number(await readFile(resolve(path, 'pid'), 'utf8'))).toBe(process.pid); await release();
+    }
+  }));
+  test('propagates suite exit and isolates all writes without building or host dependency mounts', async () => usingTemp(async (dir) => {
+    const { result, trace } = await invoke(dir, 'cli', { SUITE_EXIT: '37' });
+    expect(result.status).toBe(37);
+    expect(trace[0]).toEqual(['pull', 'node:22']);
+    const run = trace.find((args) => args[0] === 'run');
+    expect(run).toEqual(expect.arrayContaining(['--rm', '--pull=never', '--read-only', `type=bind,src=${checkout},dst=/source,readonly`, '/work:rw,exec,size=1g', '/home/test:rw,exec,size=128m']));
+    expect(trace.some((args) => args.includes('build') || args.includes('-p') || args.includes('--privileged'))).toBe(false);
+    expect(run.filter((arg) => arg.startsWith('type=bind'))).toHaveLength(1);
+    await expect(readFile(resolve(dir, 'lock/pid'))).rejects.toThrow();
+  }));
+  test('pull failure runs no container and releases the lock', async () => usingTemp(async (dir) => {
+    const { result, trace } = await invoke(dir, 'backend', { FAIL_PULL: '1' });
+    expect(result.status).toBe(1); expect(trace.some((args) => args[0] === 'run')).toBe(false);
+    await expect(readFile(resolve(dir, 'lock/pid'))).rejects.toThrow();
+  }));
+  test('service mode pulls CI databases, publishes no ports and cleans its private network', async () => usingTemp(async (dir) => {
+    const { result, trace } = await invoke(dir, 'service');
+    expect(result.status).toBe(0);
+    expect(trace.filter((args) => args[0] === 'pull').map((args) => args[1])).toEqual(['node:22', 'postgres:16', 'mongo:7']);
+    expect(trace.filter((args) => args[0] === 'run')).toHaveLength(3);
+    expect(trace.find((args) => args.includes('postgres'))).toEqual(expect.arrayContaining(['pg_isready -h 127.0.0.1 -U postgres']));
+    expect(trace.filter((args) => args[0] === 'run').every((args) => args.includes('--read-only') && args.includes('--memory') && args.includes('--rm') && !args.includes('-p'))).toBe(true);
+    expect(trace.at(-1).slice(0, 2)).toEqual(['network', 'rm']);
+  }));
+  test.each([['SIGINT', 130], ['SIGTERM', 143]])('%s cleans the container and releases the lock', async (signal, exit) => usingTemp(async (dir) => {
+    const { result, trace } = await invoke(dir, 'cli', { SIGNAL_PARENT: signal });
+    expect(result.status).toBe(exit);
+    expect(trace.some((args) => args[0] === 'rm' && args[1] === '-f')).toBe(true);
+    await expect(stat(resolve(dir, 'lock'))).rejects.toMatchObject({ code: 'ENOENT' });
+  }));
+  test('failed cleanup retains the global lock', async () => usingTemp(async (dir) => {
+    const { result } = await invoke(dir, 'cli', { FAIL_CLEANUP: '1' });
+    expect(result.status).toBe(1); expect(result.stderr).toContain('lock retained');
+    expect(Number(await readFile(resolve(dir, 'lock/pid'), 'utf8'))).toBeGreaterThan(0);
+  }));
+});

docs/development/container-tests.md

@@ -0,0 +1,96 @@
+# Disposable local test runs (D66 / TASK-507)
+
+CI remains the merge gate. This command runs tests in **pulled** CI-compatible
+images; it never builds application images. Do not substitute a deploy build.
+
+```sh
+npm run test:container -- backend
+npm run test:container -- cli
+npm run test:container -- frontend
+npm run test:container -- backend/__tests__/unit/routes/grants.test.js
+npm run test:container -- service
+# A backend/__tests__/service/*.test.js path also starts the real databases.
+```
+
+One suite or one repository-relative Jest file is accepted. `service` runs the
+backend service tier with real Mongo and Postgres. Other suites use their usual
+CI Jest setup. Tests run serially, without coverage or watch mode. The exit code
+is the suite's code (including nonzero); setup/cleanup failures return 1.
+
+## SRE policy: required before execution
+
+SRE owns the limits and the **host-global** lock path. The runner reads
+`~/.commonly/test-container-policy.json`, or the file named by
+`COMMONLY_TEST_CONTAINER_POLICY`. A missing/invalid policy refuses execution;
+there are no guessed resource defaults. SRE must confirm the Docker VM limits
+are live before the first run. A written Docker Desktop setting that has not
+been activated by a restart is not that confirmation.
+
+The policy is JSON with these required fields:
+
+| Field | Meaning |
+|---|---|
+| `lockPath` | Absolute, shared by all seats/worktrees on this host |
+| `nodeMemory`, `serviceMemory` | Docker memory caps, e.g. a positive `m`/`g` size; memory-swap equals memory |
+| `cpus`, `serviceCpus` | CPU caps for the test process and each DB |
+| `workTmpfs` | Whole writable checkout and dependency tree cap |
+| `tempTmpfs`, `homeTmpfs` | Temporary/Jest files and npm/home cache caps |
+| `serviceTmpfs` | Each DB data directory cap |
+| `logMaxSize`, `logMaxFiles` | Explicit json-file rotation limits |
+
+The SRE task note fixes Node memory at **4g**, logs at **10m × 3**, and an atomic
+mkdir lock under the operator's Commonly state directory. CPU/tmpfs/DB values
+must also come from SRE; a caller must not widen them to make a test pass.
+Policy is host configuration, not committed credentials or a repo-local lock.
+
+## What stays off the host
+
+- Source is bind mounted at `/source` **read-only**. A git-listed snapshot,
+  including unignored new files, is copied to `/work` tmpfs; tests may mutate
+  that disposable copy. Ignored host `node_modules`, `.git`, `.dev`, build and
+  coverage directories are excluded. No host dependency installation occurs.
+- Dependencies are installed fresh in `/work`; home/cache are tmpfs at
+  `/home/test`, and temp/Jest output is tmpfs under `/tmp`. No persistent
+  volumes, Docker socket, host HOME or production environment is mounted.
+- Root filesystems are read-only. DB data directories are tmpfs. Container
+  memory, CPU, process count and Docker log rotation limits are passed explicitly.
+- All containers use `--rm`, `--pull=never` after a successful explicit pull,
+  and `commonly.test=1`. They run the inspected image ID from that pull. The
+  Node major and DB tags come from `.github/workflows/tests.yml`; conflicting
+  declarations fail closed. At introduction these are Node 22, Postgres 16,
+  Mongo 7, matching CI. No `docker build` path exists.
+- Real DBs share a per-run network with the test container and publish no host
+  ports. Dependency/binary downloads still need outbound network access.
+
+The writable byte ceiling is the sum of the configured tmpfs sizes; those pages
+also consume container memory. A full tmpfs/oom is a failed run, not permission
+to use host storage. Docker image downloads live in Docker's image store and are
+bounded operationally by SRE's Docker VM disk cap/pruning. stdout is streamed;
+Docker's stored container logs are rotated at the policy cap.
+
+## Lock and cleanup
+
+The runner waits on the atomic mkdir lock, recording its PID. It never takes
+over from a live PID. A dead owner permits recovery only after its PID-labelled
+containers and networks have been removed. Signals and normal exit remove the
+current run's resources before releasing the lock. A cleanup error retains the
+lock. A partial lock without a PID, or an abandoned recovery mutex beside it,
+needs SRE inspection; do not delete a lock based on age alone.
+
+No VM restart, global prune, unrelated container stop or host temp cleanup is
+performed by the runner. Docker Desktop must already be running.
+
+## Verification of the runner
+
+The single-file Jest regression is:
+
+```sh
+npm run test:container -- cli/__tests__/container-test-runner.test.mjs
+```
+
+It uses fake Docker/git executables inside the test container to exercise exit
+propagation, pull failure, private service orchestration, limits, and cleanup
+failure without launching nested Docker. Lock tests cover waiting and dead-PID
+recovery. This is separate from a real-container smoke: check host source remains
+unchanged and no labelled container/network remains after success, failure or
+interrupt. Privileged/bwrap-specific tests are not enabled by this runner.

package.json

@@ -20,7 +20,8 @@
     "verify:numbered-rules": "node scripts/verify-numbered-rules.js",
     "verify:ax-audit-numbering": "node scripts/verify-ax-audit-numbering.js",
     "verify:readme-invariants": "node scripts/verify-readme-invariants.js && node scripts/verify-readme-invariants.js --self-test",
-    "prepare": "husky"
+    "prepare": "husky",
+    "test:container": "node scripts/container-tests/run.mjs"
   },
   "devDependencies": {
     "@playwright/test": "^1.57.0",

scripts/container-tests/inside.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+suite="$1"
+test_path="$2"
+integration="$3"
+# Only git-listed source arrives on stdin. No host node_modules, cache or .git.
+tar -C /source --null --files-from=- -cf - | tar -C /work -xf -
+cd /work
+export CI=true NODE_ENV=test HUSKY=0 JWT_SECRET=test-jwt-secret
+export SENDGRID_API_KEY=fake-api-key SENDGRID_FROM_EMAIL=test@example.com
+export FRONTEND_URL=http://localhost:3000
+npm ci --include=dev --no-audit --no-fund
+# Subprocess CLI tests import backend TS and the real MCP handler.
+npm ci --prefix backend --include=dev --no-audit --no-fund
+npm ci --prefix cli --include=dev --no-audit --no-fund
+if [[ "$suite" == cli ]]; then
+  npm ci --prefix commonly-mcp --include=dev --no-audit --no-fund
+fi
+if [[ "$suite" == frontend ]]; then
+  npm ci --prefix frontend --include=dev --legacy-peer-deps --no-audit --no-fund
+fi
+if [[ "$integration" == true ]]; then
+  export INTEGRATION_TEST=true MONGO_URI=mongodb://mongo:27017/commonly-test
+  export PG_HOST=postgres PG_PORT=5432 PG_DATABASE=commonly-test PG_USER=postgres PG_PASSWORD=postgres PG_SSL_ENABLED=false
+fi
+cd "/work/$suite"
+args=(--runInBand --forceExit --cacheDirectory=/tmp/jest-cache)
+if [[ -n "$test_path" ]]; then args+=(--runTestsByPath "$test_path");
+elif [[ "$integration" == true ]]; then args+=(--testPathPattern=__tests__/service); fi
+if [[ "$suite" == cli ]]; then
+  exec node --experimental-vm-modules node_modules/jest/bin/jest.js "${args[@]}"
+fi
+if [[ "$suite" == frontend ]]; then args+=(--watchAll=false); fi
+node node_modules/jest/bin/jest.js "${args[@]}"
+if [[ "$suite" == frontend && -z "$test_path" ]]; then
+  exec node --test scripts/generate-seo-pages.test.mjs scripts/nginx-routing.test.mjs
+fi