diff --git a/cli/__tests__/container-test-runner.test.mjs b/cli/__tests__/container-test-runner.test.mjs new file mode 100644 index 0000000000..f65ef34d69 --- /dev/null +++ b/cli/__tests__/container-test-runner.test.mjs @@ -0,0 +1,131 @@ +import { describe, test, expect } from '@jest/globals'; +import { mkdtemp, readFile, writeFile, chmod, mkdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { acquireLock, ciImages, selectSuite, validatePolicy, limits } from '../../scripts/container-tests/run.mjs'; + +const checkout = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const sample = (lockPath) => ({ lockPath, nodeMemory: '2g', serviceMemory: '1g', cpus: 1, serviceCpus: 1, + workTmpfs: '1g', tempTmpfs: '128m', homeTmpfs: '128m', serviceTmpfs: '256m', logMaxSize: '1m', logMaxFiles: 1 }); +const usingTemp = async (fn) => { + const dir = await mkdtemp(resolve(tmpdir(), 'container-runner-test-')); + try { await fn(dir); } finally { await rm(dir, { recursive: true, force: true }); } +}; +const fakeTools = async (dir) => { + const bin = resolve(dir, 'bin'); await mkdir(bin); + for (const name of ['docker', 'git']) { + const path = resolve(bin, name); + await writeFile(path, `#!/usr/bin/env node +const fs = require('fs'); +const args = process.argv.slice(2); +if (${JSON.stringify(name)} === 'git') { process.stdout.write('package.json\\0'); process.exit(0); } +fs.appendFileSync(process.env.TRACE, JSON.stringify(args)+'\\n'); +if (args[0] === 'pull' && process.env.FAIL_PULL) process.exit(17); +if (args[0] === 'image') process.stdout.write('sha256:'+args.at(-1).split(':')[0]); +if (args[0] === 'inspect') process.stdout.write('healthy'); +if (args[0] === 'rm' && process.env.FAIL_CLEANUP) process.exit(1); +if (args[0] === 'ps') process.stdout.write('container-id'); +if (args[0] === 'run' && args.includes('bash')) { + if (process.env.SIGNAL_PARENT) { process.kill(process.ppid, process.env.SIGNAL_PARENT); setTimeout(() => process.exit(0), 100); } + else process.exit(Number(process.env.SUITE_EXIT || 0)); +} +`); + await chmod(path, 0o700); + } + return bin; +}; +const invoke = async (dir, suite, extra = {}) => { + const policyFile = resolve(dir, 'policy.json'); + await writeFile(policyFile, JSON.stringify(sample(resolve(dir, 'lock')))); + const bin = await fakeTools(dir); + const result = spawnSync(process.execPath, [resolve(checkout, 'scripts/container-tests/run.mjs'), suite], { + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, TRACE: resolve(dir, 'trace'), COMMONLY_TEST_CONTAINER_POLICY: policyFile, ...extra }, + encoding: 'utf8', timeout: 15000, + }); + const trace = (await readFile(resolve(dir, 'trace'), 'utf8').catch(() => '')).trim().split('\n').filter(Boolean).map(JSON.parse); + return { result, trace }; +}; + +describe('container test runner boundaries', () => { + test('uses CI image tags and refuses conflicting versions', async () => { + const workflow = await readFile(resolve(checkout, '.github/workflows/tests.yml'), 'utf8'); + expect(ciImages(workflow)).toEqual({ node: 'node:22', postgres: 'postgres:16', mongo: 'mongo:7' }); + expect(() => ciImages(`${workflow}\nnode-version: '24'`)).toThrow('disagree'); + }); + test('requires every SRE limit before execution', () => { + expect(() => validatePolicy({})).toThrow('lockPath'); + for (const key of Object.keys(sample('/tmp/test-lock'))) { + const policy = sample('/tmp/test-lock'); delete policy[key]; + expect(() => validatePolicy(policy)).toThrow(); + } + expect(limits(validatePolicy(sample('/tmp/test-lock')), '2g')).toEqual(expect.arrayContaining([ + '--read-only', '--memory', '2g', '--memory-swap', '--cpus', '1', '--log-driver', 'json-file', 'max-size=1m', 'max-file=1', + ])); + }); + test('selects exact Jest files and refuses traversal or arbitrary commands', () => { + expect(selectSuite(['backend/__tests__/service/foo.test.js'])).toEqual({ suite: 'backend', integration: true, path: '__tests__/service/foo.test.js' }); + for (const input of ['../../oops', 'cli/../backend/a.test.js', '--privileged', 'backend;echo hacked', '/tmp/a.test.js']) { + expect(() => selectSuite([input])).toThrow(); + } + expect(() => selectSuite(['cli', '--watch'])).toThrow(); + }); + test('lock contention waits for the active owner, then acquires', async () => usingTemp(async (dir) => { + const path = resolve(dir, 'lock'); const release = await acquireLock(path); + let waits = 0; + const nextRelease = await acquireLock(path, { wait: async () => { waits += 1; await release(); } }); + expect(waits).toBe(1); + expect(Number(await readFile(resolve(path, 'pid'), 'utf8'))).toBe(process.pid); + await nextRelease(); + })); + test('dead PID is the only automatic stale-lock recovery', async () => usingTemp(async (dir) => { + const path = resolve(dir, 'lock'); + for (let attempt = 0; attempt < 2; attempt += 1) { + await mkdir(path); await writeFile(resolve(path, 'pid'), '2147483647'); + const release = await acquireLock(path, { alive: () => false, onStale: async (pid) => { + expect(pid).toBe(2147483647); + // Cleanup must finish while the old lock still excludes other runs. + expect(await readFile(resolve(path, 'pid'), 'utf8')).toBe(String(pid)); + expect((await stat(`${path}.reap`)).isDirectory()).toBe(true); + }, wait: async () => { throw new Error('recovery mutex leaked'); } }); + await expect(stat(`${path}.reap`)).rejects.toMatchObject({ code: 'ENOENT' }); + expect(Number(await readFile(resolve(path, 'pid'), 'utf8'))).toBe(process.pid); await release(); + } + })); + test('propagates suite exit and isolates all writes without building or host dependency mounts', async () => usingTemp(async (dir) => { + const { result, trace } = await invoke(dir, 'cli', { SUITE_EXIT: '37' }); + expect(result.status).toBe(37); + expect(trace[0]).toEqual(['pull', 'node:22']); + const run = trace.find((args) => args[0] === 'run'); + expect(run).toEqual(expect.arrayContaining(['--rm', '--pull=never', '--read-only', `type=bind,src=${checkout},dst=/source,readonly`, '/work:rw,exec,size=1g', '/home/test:rw,exec,size=128m'])); + expect(trace.some((args) => args.includes('build') || args.includes('-p') || args.includes('--privileged'))).toBe(false); + expect(run.filter((arg) => arg.startsWith('type=bind'))).toHaveLength(1); + await expect(readFile(resolve(dir, 'lock/pid'))).rejects.toThrow(); + })); + test('pull failure runs no container and releases the lock', async () => usingTemp(async (dir) => { + const { result, trace } = await invoke(dir, 'backend', { FAIL_PULL: '1' }); + expect(result.status).toBe(1); expect(trace.some((args) => args[0] === 'run')).toBe(false); + await expect(readFile(resolve(dir, 'lock/pid'))).rejects.toThrow(); + })); + test('service mode pulls CI databases, publishes no ports and cleans its private network', async () => usingTemp(async (dir) => { + const { result, trace } = await invoke(dir, 'service'); + expect(result.status).toBe(0); + expect(trace.filter((args) => args[0] === 'pull').map((args) => args[1])).toEqual(['node:22', 'postgres:16', 'mongo:7']); + expect(trace.filter((args) => args[0] === 'run')).toHaveLength(3); + expect(trace.find((args) => args.includes('postgres'))).toEqual(expect.arrayContaining(['pg_isready -h 127.0.0.1 -U postgres'])); + expect(trace.filter((args) => args[0] === 'run').every((args) => args.includes('--read-only') && args.includes('--memory') && args.includes('--rm') && !args.includes('-p'))).toBe(true); + expect(trace.at(-1).slice(0, 2)).toEqual(['network', 'rm']); + })); + test.each([['SIGINT', 130], ['SIGTERM', 143]])('%s cleans the container and releases the lock', async (signal, exit) => usingTemp(async (dir) => { + const { result, trace } = await invoke(dir, 'cli', { SIGNAL_PARENT: signal }); + expect(result.status).toBe(exit); + expect(trace.some((args) => args[0] === 'rm' && args[1] === '-f')).toBe(true); + await expect(stat(resolve(dir, 'lock'))).rejects.toMatchObject({ code: 'ENOENT' }); + })); + test('failed cleanup retains the global lock', async () => usingTemp(async (dir) => { + const { result } = await invoke(dir, 'cli', { FAIL_CLEANUP: '1' }); + expect(result.status).toBe(1); expect(result.stderr).toContain('lock retained'); + expect(Number(await readFile(resolve(dir, 'lock/pid'), 'utf8'))).toBeGreaterThan(0); + })); +}); diff --git a/docs/development/container-tests.md b/docs/development/container-tests.md new file mode 100644 index 0000000000..a68e2aa00c --- /dev/null +++ b/docs/development/container-tests.md @@ -0,0 +1,96 @@ +# Disposable local test runs (D66 / TASK-507) + +CI remains the merge gate. This command runs tests in **pulled** CI-compatible +images; it never builds application images. Do not substitute a deploy build. + +```sh +npm run test:container -- backend +npm run test:container -- cli +npm run test:container -- frontend +npm run test:container -- backend/__tests__/unit/routes/grants.test.js +npm run test:container -- service +# A backend/__tests__/service/*.test.js path also starts the real databases. +``` + +One suite or one repository-relative Jest file is accepted. `service` runs the +backend service tier with real Mongo and Postgres. Other suites use their usual +CI Jest setup. Tests run serially, without coverage or watch mode. The exit code +is the suite's code (including nonzero); setup/cleanup failures return 1. + +## SRE policy: required before execution + +SRE owns the limits and the **host-global** lock path. The runner reads +`~/.commonly/test-container-policy.json`, or the file named by +`COMMONLY_TEST_CONTAINER_POLICY`. A missing/invalid policy refuses execution; +there are no guessed resource defaults. SRE must confirm the Docker VM limits +are live before the first run. A written Docker Desktop setting that has not +been activated by a restart is not that confirmation. + +The policy is JSON with these required fields: + +| Field | Meaning | +|---|---| +| `lockPath` | Absolute, shared by all seats/worktrees on this host | +| `nodeMemory`, `serviceMemory` | Docker memory caps, e.g. a positive `m`/`g` size; memory-swap equals memory | +| `cpus`, `serviceCpus` | CPU caps for the test process and each DB | +| `workTmpfs` | Whole writable checkout and dependency tree cap | +| `tempTmpfs`, `homeTmpfs` | Temporary/Jest files and npm/home cache caps | +| `serviceTmpfs` | Each DB data directory cap | +| `logMaxSize`, `logMaxFiles` | Explicit json-file rotation limits | + +The SRE task note fixes Node memory at **4g**, logs at **10m × 3**, and an atomic +mkdir lock under the operator's Commonly state directory. CPU/tmpfs/DB values +must also come from SRE; a caller must not widen them to make a test pass. +Policy is host configuration, not committed credentials or a repo-local lock. + +## What stays off the host + +- Source is bind mounted at `/source` **read-only**. A git-listed snapshot, + including unignored new files, is copied to `/work` tmpfs; tests may mutate + that disposable copy. Ignored host `node_modules`, `.git`, `.dev`, build and + coverage directories are excluded. No host dependency installation occurs. +- Dependencies are installed fresh in `/work`; home/cache are tmpfs at + `/home/test`, and temp/Jest output is tmpfs under `/tmp`. No persistent + volumes, Docker socket, host HOME or production environment is mounted. +- Root filesystems are read-only. DB data directories are tmpfs. Container + memory, CPU, process count and Docker log rotation limits are passed explicitly. +- All containers use `--rm`, `--pull=never` after a successful explicit pull, + and `commonly.test=1`. They run the inspected image ID from that pull. The + Node major and DB tags come from `.github/workflows/tests.yml`; conflicting + declarations fail closed. At introduction these are Node 22, Postgres 16, + Mongo 7, matching CI. No `docker build` path exists. +- Real DBs share a per-run network with the test container and publish no host + ports. Dependency/binary downloads still need outbound network access. + +The writable byte ceiling is the sum of the configured tmpfs sizes; those pages +also consume container memory. A full tmpfs/oom is a failed run, not permission +to use host storage. Docker image downloads live in Docker's image store and are +bounded operationally by SRE's Docker VM disk cap/pruning. stdout is streamed; +Docker's stored container logs are rotated at the policy cap. + +## Lock and cleanup + +The runner waits on the atomic mkdir lock, recording its PID. It never takes +over from a live PID. A dead owner permits recovery only after its PID-labelled +containers and networks have been removed. Signals and normal exit remove the +current run's resources before releasing the lock. A cleanup error retains the +lock. A partial lock without a PID, or an abandoned recovery mutex beside it, +needs SRE inspection; do not delete a lock based on age alone. + +No VM restart, global prune, unrelated container stop or host temp cleanup is +performed by the runner. Docker Desktop must already be running. + +## Verification of the runner + +The single-file Jest regression is: + +```sh +npm run test:container -- cli/__tests__/container-test-runner.test.mjs +``` + +It uses fake Docker/git executables inside the test container to exercise exit +propagation, pull failure, private service orchestration, limits, and cleanup +failure without launching nested Docker. Lock tests cover waiting and dead-PID +recovery. This is separate from a real-container smoke: check host source remains +unchanged and no labelled container/network remains after success, failure or +interrupt. Privileged/bwrap-specific tests are not enabled by this runner. diff --git a/package.json b/package.json index 92de278f15..fc70f9f5e9 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "verify:numbered-rules": "node scripts/verify-numbered-rules.js", "verify:ax-audit-numbering": "node scripts/verify-ax-audit-numbering.js", "verify:readme-invariants": "node scripts/verify-readme-invariants.js && node scripts/verify-readme-invariants.js --self-test", - "prepare": "husky" + "prepare": "husky", + "test:container": "node scripts/container-tests/run.mjs" }, "devDependencies": { "@playwright/test": "^1.57.0", diff --git a/scripts/container-tests/inside.sh b/scripts/container-tests/inside.sh new file mode 100644 index 0000000000..99692b4a33 --- /dev/null +++ b/scripts/container-tests/inside.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +suite="$1" +test_path="$2" +integration="$3" +# Only git-listed source arrives on stdin. No host node_modules, cache or .git. +tar -C /source --null --files-from=- -cf - | tar -C /work -xf - +cd /work +export CI=true NODE_ENV=test HUSKY=0 JWT_SECRET=test-jwt-secret +export SENDGRID_API_KEY=fake-api-key SENDGRID_FROM_EMAIL=test@example.com +export FRONTEND_URL=http://localhost:3000 +npm ci --include=dev --no-audit --no-fund +# Subprocess CLI tests import backend TS and the real MCP handler. +npm ci --prefix backend --include=dev --no-audit --no-fund +npm ci --prefix cli --include=dev --no-audit --no-fund +if [[ "$suite" == cli ]]; then + npm ci --prefix commonly-mcp --include=dev --no-audit --no-fund +fi +if [[ "$suite" == frontend ]]; then + npm ci --prefix frontend --include=dev --legacy-peer-deps --no-audit --no-fund +fi +if [[ "$integration" == true ]]; then + export INTEGRATION_TEST=true MONGO_URI=mongodb://mongo:27017/commonly-test + export PG_HOST=postgres PG_PORT=5432 PG_DATABASE=commonly-test PG_USER=postgres PG_PASSWORD=postgres PG_SSL_ENABLED=false +fi +cd "/work/$suite" +args=(--runInBand --forceExit --cacheDirectory=/tmp/jest-cache) +if [[ -n "$test_path" ]]; then args+=(--runTestsByPath "$test_path"); +elif [[ "$integration" == true ]]; then args+=(--testPathPattern=__tests__/service); fi +if [[ "$suite" == cli ]]; then + exec node --experimental-vm-modules node_modules/jest/bin/jest.js "${args[@]}" +fi +if [[ "$suite" == frontend ]]; then args+=(--watchAll=false); fi +node node_modules/jest/bin/jest.js "${args[@]}" +if [[ "$suite" == frontend && -z "$test_path" ]]; then + exec node --test scripts/generate-seo-pages.test.mjs scripts/nginx-routing.test.mjs +fi diff --git a/scripts/container-tests/run.mjs b/scripts/container-tests/run.mjs new file mode 100644 index 0000000000..2b700ab6fe --- /dev/null +++ b/scripts/container-tests/run.mjs @@ -0,0 +1,176 @@ +#!/usr/bin/env node +import { spawn } from 'node:child_process'; +import { lstat, mkdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, isAbsolute, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { randomBytes } from 'node:crypto'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const size = /^[1-9]\d*[kmg]$/i; +export function validatePolicy(value) { + if (!value || !isAbsolute(value.lockPath || '') || value.lockPath === '/') throw new Error('SRE policy needs an absolute lockPath'); + for (const key of ['nodeMemory', 'serviceMemory', 'workTmpfs', 'tempTmpfs', 'homeTmpfs', 'serviceTmpfs', 'logMaxSize']) { + if (!size.test(value[key] || '')) throw new Error(`SRE policy needs ${key} (positive k/m/g size)`); + } + if (!(value.cpus > 0 && value.cpus <= 64 && value.serviceCpus > 0 && value.serviceCpus <= 64) || !Number.isInteger(value.logMaxFiles) || value.logMaxFiles < 1 || value.logMaxFiles > 10) { + throw new Error('SRE policy needs positive cpus (<=64) and logMaxFiles (1..10)'); + } + return value; +} +export function selectSuite(args) { + if (args.length !== 1) throw new Error('Usage: npm run test:container -- backend|cli|frontend|service|'); + const name = args[0]; + if (['backend', 'cli', 'frontend', 'service'].includes(name)) return { suite: name === 'service' ? 'backend' : name, integration: name === 'service', path: '' }; + const match = /^(backend|cli|frontend)\/(.+\.(?:test|spec)\.[cm]?[jt]sx?)$/.exec(name); + if (!match || name.split('/').some((part) => part === '..' || part === '.')) throw new Error('Expected a suite or a repo-relative Jest test file'); + return { suite: match[1], integration: name.startsWith('backend/__tests__/service/'), path: match[2] }; +} +export function ciImages(workflow) { + const versions = [...workflow.matchAll(/node-version:\s*['"]?(\d+)/g)].map((m) => m[1]); + const tags = (name) => [...new Set([...workflow.matchAll(new RegExp(`image: (${name}:[\\w.-]+)`, 'g'))].map((m) => m[1]))]; + if (!versions.length || new Set(versions).size !== 1) throw new Error('CI Node versions disagree'); + const postgres = tags('postgres'); const mongo = tags('mongo'); + if (postgres.length !== 1 || mongo.length !== 1) throw new Error('CI database image tags disagree or are missing'); + return { node: `node:${versions[0]}`, postgres: postgres[0], mongo: mongo[0] }; +} +export function limits(policy, memory, cpus = policy.cpus) { + return ['--read-only', '--memory', memory, '--memory-swap', memory, '--cpus', String(cpus), + '--pids-limit', '512', '--log-driver', 'json-file', '--log-opt', `max-size=${policy.logMaxSize}`, + '--log-opt', `max-file=${policy.logMaxFiles}`, '--tmpfs', `/tmp:rw,exec,size=${policy.tempTmpfs}`]; +} +function command(program, args, { input, capture = false } = {}) { + return new Promise((accept, reject) => { + const child = spawn(program, args, { stdio: [input === undefined ? 'ignore' : 'pipe', capture ? 'pipe' : 'inherit', 'inherit'] }); + const chunks = []; + if (capture) child.stdout.on('data', (data) => chunks.push(data)); + child.on('error', reject); + child.on('close', (code, signal) => accept({ code: code ?? (signal === 'SIGINT' ? 130 : 143), output: Buffer.concat(chunks).toString() })); + if (input !== undefined) { child.stdin.on('error', () => {}); child.stdin.end(input); } + }); +} +async function checked(program, args, options) { + const result = await command(program, args, options); + if (result.code !== 0) throw new Error(`${program} ${args[0]} failed (${result.code})`); + return result.output.trim(); +} +export async function acquireLock(path, { onStale = async () => {}, wait = () => new Promise((done) => setTimeout(done, 1000)), + alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } }, +} = {}) { + await mkdir(dirname(path), { recursive: true }); + for (;;) { + try { await mkdir(path, { mode: 0o700 }); break; } + catch (error) { if (error.code !== 'EEXIST') throw error; } + // Re-read under a recovery mutex: two waiters must never remove a new owner. + let recovering = false; + try { + await mkdir(`${path}.reap`); recovering = true; + const pid = Number((await readFile(resolve(path, 'pid'), 'utf8')).trim()); + if (Number.isInteger(pid) && pid > 0 && !alive(pid)) { + await onStale(pid); + await rm(path, { recursive: true }); + continue; + } + } catch (error) { + if (!['EEXIST', 'ENOENT'].includes(error.code)) throw error; + } finally { if (recovering) await rm(`${path}.reap`, { recursive: true, force: true }); } + await wait(); + } + try { await writeFile(resolve(path, 'pid'), String(process.pid), { mode: 0o600 }); } + catch (error) { await rm(path, { recursive: true }); throw error; } + return () => rm(path, { recursive: true }); +} + +async function sourceManifest() { + const result = await checked('git', ['-C', root, 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], { capture: true }); + const entries = []; + for (const path of new Set(result.split('\0').filter(Boolean))) { + if (path.split('/').some((part) => ['node_modules', '.git', 'dist', 'coverage', '.dev'].includes(part))) continue; + const info = await lstat(resolve(root, path)).catch(() => null); + if (info && !info.isDirectory()) entries.push(path); + } + return Buffer.from(`${entries.join('\0')}\0`); +} +export async function main(args) { + const target = selectSuite(args); + if (target.path) { + const file = await realpath(resolve(root, target.suite, target.path)); + if (relative(root, file).startsWith('..')) throw new Error('Test path escapes the checkout'); + } + const policyPath = process.env.COMMONLY_TEST_CONTAINER_POLICY || resolve(homedir(), '.commonly/test-container-policy.json'); + let policy; + try { policy = validatePolicy(JSON.parse(await readFile(policyPath, 'utf8'))); } + catch (error) { throw new Error(`SRE policy unavailable/invalid at ${policyPath}: ${error.message}. Request approved caps and lockPath on TASK-507.`); } + const images = ciImages(await readFile(resolve(root, '.github/workflows/tests.yml'), 'utf8')); + if (root.includes(',')) throw new Error('Docker source mount requires a checkout path without commas'); + const release = await acquireLock(policy.lockPath, { onStale: async (pid) => { + const filter = `label=commonly.test.pid=${pid}`; + const names = await checked('docker', ['ps', '-aq', '--filter', filter], { capture: true }); + if (names) await checked('docker', ['rm', '-f', ...names.split(/\s+/)], { capture: true }); + const networks = await checked('docker', ['network', 'ls', '-q', '--filter', filter], { capture: true }); + if (networks) await checked('docker', ['network', 'rm', ...networks.split(/\s+/)], { capture: true }); + } }); + const id = `commonly-test-${process.pid}-${randomBytes(4).toString('hex')}`; + const containers = []; let network = false; let cleaning; + const cleanup = () => { + if (cleaning) return cleaning; + cleaning = (async () => { + let failed = false; + for (const name of containers.reverse()) { + const remaining = await command('docker', ['ps', '-a', '--format', '{{.ID}}', '--filter', `name=^${name}$`], { capture: true }); + if (remaining.code) failed = true; + else if (remaining.output.trim() && (await command('docker', ['rm', '-f', name], { capture: true })).code) failed = true; + } + if (network && (await command('docker', ['network', 'rm', id], { capture: true })).code) failed = true; + if (failed) throw new Error(`Cleanup incomplete; lock retained at ${policy.lockPath}. Ask SRE to inspect ${id}.`); + await release(); + })(); + return cleaning; + }; + const onSignal = async (signal) => { + try { await cleanup(); } catch (error) { console.error(error.message); } + process.exit(signal === 'SIGINT' ? 130 : 143); + }; + const interrupt = () => { void onSignal('SIGINT'); }; const terminate = () => { void onSignal('SIGTERM'); }; + process.once('SIGINT', interrupt); process.once('SIGTERM', terminate); + try { + const needed = target.integration ? ['node', 'postgres', 'mongo'] : ['node']; + const pulled = {}; + for (const key of needed) { + await checked('docker', ['pull', images[key]]); + pulled[key] = await checked('docker', ['image', 'inspect', '--format', '{{.Id}}', images[key]], { capture: true }); + } + if (target.integration) { + await checked('docker', ['network', 'create', '--label', `commonly.test.pid=${process.pid}`, '--label', 'commonly.test=1', id], { capture: true }); network = true; + for (const key of ['postgres', 'mongo']) { + const name = `${id}-${key}`; containers.push(name); + const dbArgs = key === 'postgres' + ? ['--tmpfs', `/var/lib/postgresql/data:rw,size=${policy.serviceTmpfs}`, '--tmpfs', `/var/run/postgresql:rw,size=${policy.tempTmpfs}`, '-e', 'POSTGRES_PASSWORD=postgres', '-e', 'POSTGRES_DB=commonly-test', '--health-cmd', 'pg_isready -h 127.0.0.1 -U postgres'] + : ['--tmpfs', `/data/db:rw,size=${policy.serviceTmpfs}`, '--tmpfs', `/data/configdb:rw,size=${policy.serviceTmpfs}`, '--ulimit', 'nofile=64000:64000', '--health-cmd', 'mongosh --quiet --eval "db.runCommand({ping:1}).ok"']; + await checked('docker', ['run', '--rm', '--pull=never', '-d', '--name', name, '--label', 'commonly.test=1', '--label', `commonly.test.pid=${process.pid}`, '--network', id, '--network-alias', key, + ...limits(policy, policy.serviceMemory, policy.serviceCpus), ...dbArgs, '--health-interval', '1s', '--health-timeout', '5s', '--health-retries', '60', pulled[key]], { capture: true }); + let ready = false; + for (let attempt = 0; attempt < 60; attempt += 1) { + const state = await checked('docker', ['inspect', '--format', '{{.State.Health.Status}}', name], { capture: true }); + if (state === 'healthy') { ready = true; break; } + if (state === 'unhealthy') break; + await new Promise((done) => setTimeout(done, 1000)); + } + if (!ready) throw new Error(`${key} failed its readiness check`); + } + } + const name = `${id}-node`; containers.push(name); + const result = await command('docker', ['run', '--rm', '--pull=never', '-i', '--name', name, '--label', 'commonly.test=1', '--label', `commonly.test.pid=${process.pid}`, + ...limits(policy, policy.nodeMemory), ...(network ? ['--network', id] : []), + '--mount', `type=bind,src=${root},dst=/source,readonly`, + '--tmpfs', `/work:rw,exec,size=${policy.workTmpfs}`, '--tmpfs', `/home/test:rw,exec,size=${policy.homeTmpfs}`, + '-e', 'HOME=/home/test', '-e', 'TMPDIR=/tmp', '-e', 'npm_config_cache=/home/test/.npm', '-e', 'XDG_CACHE_HOME=/home/test/.cache', + pulled.node, 'bash', '/source/scripts/container-tests/inside.sh', target.suite, target.path, String(target.integration)], { input: await sourceManifest() }); + return result.code; + } finally { + await cleanup(); process.removeListener('SIGINT', interrupt); process.removeListener('SIGTERM', terminate); + } +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => { console.error(error.message); process.exitCode = 1; }); +}