diff --git a/quest/m0/gate-test-leak.md b/quest/m0/gate-test-leak.md deleted file mode 100644 index 5bcacfba..00000000 --- a/quest/m0/gate-test-leak.md +++ /dev/null @@ -1,27 +0,0 @@ -# [XS] The gate's nested-scope tests leave no scope running - -## Goal - -`just test` leaves no `quest-gate-*.scope` running after it finishes, -pass or fail. - -## Plan - -Found 2026-11-12 on the maintainer's host: two gate scopes from a test -run on 2026-11-11 at 00:57 were still running 42 hours later, one holding -`just --justfile /tmp/quest-test-KEYeCs/justfile ... phase` and the other -`bun /tmp/quest-test-KEYeCs/nested.ts`. They match `test/gate.test.ts`'s -`nestedGate` and `phase` helpers, the tests around "is stopped when the -enclosing scope's owner kills it". Both were stopped by hand on -2026-11-12, so reproduce the leak before fixing it, per the rule. - -Decided 2026-11-12 (maintainer, /quest-plan): ✅ plan it in m0 -(rejected: m1). - -Fix it at the root: a leaked scope means some path, a failed or -timed-out test included, ends without stopping the scope it bound, which -may also leak in real nested gates outside the tests. - -## Hosts - -- linux - only systemd scopes can leak diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index feac04a2..7bde5a57 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -210,7 +210,7 @@ function signalGroup(group: string, signal: NodeJS.Signals): void { } /** The processes in the cgroup and every cgroup below it; none once it is removed. */ -function members(group: string): number[] { +export function members(group: string): number[] { try { const own = readFileSync(join(group, "cgroup.procs"), "utf8").split("\n").filter(Boolean).map(Number); const children = readdirSync(group, { withFileTypes: true }).filter((entry) => entry.isDirectory()); diff --git a/test/scopes.test.ts b/test/scopes.test.ts new file mode 100644 index 00000000..750f5d86 --- /dev/null +++ b/test/scopes.test.ts @@ -0,0 +1,70 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { hostname } from "node:os"; +import { join } from "node:path"; + +import { describe, expect, test } from "vitest"; + +import { alive, startTime, systemdScopes } from "../src/cli/supervise"; +import { created, tempDir } from "./fixture"; +import { RUN, sweep } from "./scopes"; + +describe.runIf(process.platform === "linux")("the test run's scope sweep", () => { + test("marks every test process with the live run it belongs to", () => { + const [pid, start] = process.env[RUN]!.split(":"); + expect(alive({ host: hostname(), pid: Number(pid), start })).toBe(true); + }); + + /** Start `sleep` in a new Quest scope as a process of `run`, resolving once it runs there. */ + async function leave(run: string) { + const dir = tempDir(); + const ready = join(dir, "ready"); + const unit = `quest-gate-${randomUUID()}.scope`; + const child = spawn( + "systemd-run", + [ + "--user", + "--scope", + "--quiet", + "--collect", + `--unit=${unit}`, + "/bin/sh", + "-c", + `: > ${JSON.stringify(ready)}; exec sleep 1000`, + ], + { env: { ...process.env, [RUN]: run }, stdio: "ignore" }, + ); + const exited = once(child, "exit"); + // A launcher that exits first, such as one no user manager answers, fails the test at once. + await new Promise((resolve, reject) => { + created(ready).then(resolve, reject); + exited.then( + ([code, signal]) => reject(new Error(`systemd-run ended (${code ?? signal}) before its command ran`)), + reject, + ); + }); + return { unit, exited }; + } + + test("stops a scope a dead run left and spares one of a live run", async () => { + // This process's PID with a start time it never had names a run that has exited. + const left = await leave(`${process.pid}:0`); + const kept = await leave(process.env[RUN]!); + await sweep(); + expect(systemdScopes.live(left.unit)).toBe(false); + expect(systemdScopes.live(kept.unit)).toBe(true); + expect(await left.exited).toEqual([null, "SIGKILL"]); + await systemdScopes.stop(kept.unit); + expect(await kept.exited).toEqual([null, "SIGKILL"]); + }); + + test("stops the scopes of the live run it names", async () => { + // This worker names a live run other than the one every test belongs to. + const run = `${process.pid}:${startTime(process.pid)}`; + const left = await leave(run); + await sweep(run); + expect(systemdScopes.live(left.unit)).toBe(false); + expect(await left.exited).toEqual([null, "SIGKILL"]); + }); +}); diff --git a/test/scopes.ts b/test/scopes.ts new file mode 100644 index 00000000..2323f926 --- /dev/null +++ b/test/scopes.ts @@ -0,0 +1,110 @@ +// Stops the Quest scopes a test run leaves running. A scope outlives the process that started it, +// so a test that fails or times out before stopping one, a supervisor a test kills, or a run +// signalled mid-test (a cancelled `quest gate` signals every process in the run) would leave it. +// Every process the run starts, child CLIs and their scopes included, inherits the run's marker in +// its environment, so the sweep finds a scope by what runs in it rather than by who started it. + +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { hostname } from "node:os"; +import { join } from "node:path"; + +import { alive, kill, members, startTime, systemdScopes } from "../src/cli/supervise"; + +/** Names the test run a process belongs to, as its main process's `:`. */ +export const RUN = "QUEST_TEST_RUN"; + +/** Whether the test run `run` names has exited. */ +function dead(run: string): boolean { + const match = /^(\d+):(\d+)$/.exec(run); + if (!match) throw new Error(`malformed ${RUN}: ${run}`); + return !alive({ host: hostname(), pid: Number(match[1]), start: match[2] }); +} + +/** + * The run `pid` belongs to, or undefined for a process outside every run, already gone, or hiding + * its environment, as a setuid program another session runs does; no test runs one. + */ +function runOf(pid: number): string | undefined { + try { + const environ = readFileSync(`/proc/${pid}/environ`, "utf8").split("\0"); + return environ.find((entry) => entry.startsWith(`${RUN}=`))?.slice(RUN.length + 1); + } catch (error) { + if (["ENOENT", "ESRCH", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) return undefined; + throw error; + } +} + +/** + * The cgroups of the Quest scopes holding a process of run `own` or of a run that has exited, but + * never one holding this process, such as the gate's scope around `just test`. + */ +function strays(own?: string): Map { + const self = readFileSync("/proc/self/cgroup", "utf8") + .split("\n") + .find((line) => line.startsWith("0::"))! + .slice(3); + const show = execFileSync("systemctl", ["--user", "show", "--property=Id,ControlGroup", "quest-*.scope"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + const found = new Map(); + for (const entry of show.split("\n\n")) { + const unit = /^Id=(.+)$/m.exec(entry)?.[1]; + const group = /^ControlGroup=(.+)$/m.exec(entry)?.[1]; + if (!unit || !group || self === group || self.startsWith(`${group}/`)) continue; + const path = join("/sys/fs/cgroup", group); + const runs = members(path).map(runOf); + if (runs.some((run) => run !== undefined && (run === own || dead(run)))) found.set(unit, path); + } + return found; +} + +/** Stop every scope a run that exited left running, and those of run `own`; resolves once each is empty. */ +export async function sweep(own?: string): Promise { + const stops = await Promise.allSettled([...strays(own).keys()].map((unit) => systemdScopes.stop(unit))); + const failed = stops.flatMap((stop) => (stop.status === "rejected" ? [stop.reason] : [])); + if (failed.length) throw new AggregateError(failed, `cannot stop ${failed.length} test scopes`); +} + +/** Whether a user manager answers; without one, no test can start a scope, and none was left. */ +function managed(): boolean { + try { + execFileSync("systemctl", ["--user", "show", "--property=Version", "--value"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** Vitest's global setup: marks the run, clears what dead runs left, and stops this run's scopes when it ends. */ +export default async function setup(): Promise<(() => Promise) | undefined> { + if (process.platform !== "linux") return undefined; + const own = `${process.pid}:${startTime(process.pid)}`; + // Set before the workers start, so they and everything they spawn inherit it. + process.env[RUN] = own; + // Such as in a container: the tests that start scopes fail there on their own, and the rest still run. + if (!managed()) return undefined; + // A run killed outright, such as by SIGKILL, never reached its teardown. + await sweep(); + // A signalled run never reaches its teardown either. + for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + // One scope that cannot be killed must not spare the rest, nor keep the run alive. + try { + for (const [unit, group] of strays(own)) + try { + kill(group); + } catch (error) { + process.stderr.write(`cannot kill test scope ${unit}: ${error}\n`); + } + } catch (error) { + process.stderr.write(`cannot find the test scopes: ${error}\n`); + } + // Raised again once this listener is gone, the signal ends the run as it would have without + // it: signal-exit, which Vite's bundler loads, re-raises a signal only when it listens alone. + setImmediate(() => process.kill(process.pid, signal)); + }); + } + return () => sweep(own); +} diff --git a/vitest.config.ts b/vitest.config.ts index a5757b30..3d66f802 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -28,6 +28,7 @@ export default defineConfig({ // A test's CPU budget (test/budget.ts) bounds how slow it may be; this wall-clock // limit only catches a test awaiting what never comes, so a loaded host never reaches it. setupFiles: ["test/budget.ts"], + globalSetup: ["test/scopes.ts"], testTimeout: 5 * 60_000, hookTimeout: 5 * 60_000, }),