Skip to content
Merged
27 changes: 0 additions & 27 deletions quest/m0/gate-test-leak.md

This file was deleted.

2 changes: 1 addition & 1 deletion src/cli/supervise.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ function signalGroup(group: string, signal: NodeJS.Signals): void {
}

/** The processes in the cgroup and every cgroup below it; none once it is removed. */
function members(group: string): number[] {
export function members(group: string): number[] {
try {
const own = readFileSync(join(group, "cgroup.procs"), "utf8").split("\n").filter(Boolean).map(Number);
const children = readdirSync(group, { withFileTypes: true }).filter((entry) => entry.isDirectory());
Expand Down
70 changes: 70 additions & 0 deletions test/scopes.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { once } from "node:events";
import { hostname } from "node:os";
import { join } from "node:path";

import { describe, expect, test } from "vitest";

import { alive, startTime, systemdScopes } from "../src/cli/supervise";
import { created, tempDir } from "./fixture";
import { RUN, sweep } from "./scopes";

describe.runIf(process.platform === "linux")("the test run's scope sweep", () => {
test("marks every test process with the live run it belongs to", () => {
const [pid, start] = process.env[RUN]!.split(":");
expect(alive({ host: hostname(), pid: Number(pid), start })).toBe(true);
});

/** Start `sleep` in a new Quest scope as a process of `run`, resolving once it runs there. */
async function leave(run: string) {
const dir = tempDir();
const ready = join(dir, "ready");
const unit = `quest-gate-${randomUUID()}.scope`;
const child = spawn(
"systemd-run",
[
"--user",
"--scope",
"--quiet",
"--collect",
`--unit=${unit}`,
"/bin/sh",
"-c",
`: > ${JSON.stringify(ready)}; exec sleep 1000`,
],
{ env: { ...process.env, [RUN]: run }, stdio: "ignore" },
);
const exited = once(child, "exit");
// A launcher that exits first, such as one no user manager answers, fails the test at once.
await new Promise<void>((resolve, reject) => {
created(ready).then(resolve, reject);
exited.then(
([code, signal]) => reject(new Error(`systemd-run ended (${code ?? signal}) before its command ran`)),
reject,
);
});
return { unit, exited };
}

test("stops a scope a dead run left and spares one of a live run", async () => {
// This process's PID with a start time it never had names a run that has exited.
const left = await leave(`${process.pid}:0`);
const kept = await leave(process.env[RUN]!);
await sweep();
expect(systemdScopes.live(left.unit)).toBe(false);
expect(systemdScopes.live(kept.unit)).toBe(true);
expect(await left.exited).toEqual([null, "SIGKILL"]);
await systemdScopes.stop(kept.unit);
expect(await kept.exited).toEqual([null, "SIGKILL"]);
});

test("stops the scopes of the live run it names", async () => {
// This worker names a live run other than the one every test belongs to.
const run = `${process.pid}:${startTime(process.pid)}`;
const left = await leave(run);
await sweep(run);
expect(systemdScopes.live(left.unit)).toBe(false);
expect(await left.exited).toEqual([null, "SIGKILL"]);
});
});
110 changes: 110 additions & 0 deletions test/scopes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Stops the Quest scopes a test run leaves running. A scope outlives the process that started it,
// so a test that fails or times out before stopping one, a supervisor a test kills, or a run
// signalled mid-test (a cancelled `quest gate` signals every process in the run) would leave it.
// Every process the run starts, child CLIs and their scopes included, inherits the run's marker in
// its environment, so the sweep finds a scope by what runs in it rather than by who started it.

import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import { hostname } from "node:os";
import { join } from "node:path";

import { alive, kill, members, startTime, systemdScopes } from "../src/cli/supervise";

/** Names the test run a process belongs to, as its main process's `<pid>:<start time>`. */
export const RUN = "QUEST_TEST_RUN";

/** Whether the test run `run` names has exited. */
function dead(run: string): boolean {
const match = /^(\d+):(\d+)$/.exec(run);
if (!match) throw new Error(`malformed ${RUN}: ${run}`);
return !alive({ host: hostname(), pid: Number(match[1]), start: match[2] });
}

/**
* The run `pid` belongs to, or undefined for a process outside every run, already gone, or hiding
* its environment, as a setuid program another session runs does; no test runs one.
*/
function runOf(pid: number): string | undefined {
try {
const environ = readFileSync(`/proc/${pid}/environ`, "utf8").split("\0");
return environ.find((entry) => entry.startsWith(`${RUN}=`))?.slice(RUN.length + 1);
} catch (error) {
if (["ENOENT", "ESRCH", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) return undefined;
throw error;
}
}

/**
* The cgroups of the Quest scopes holding a process of run `own` or of a run that has exited, but
* never one holding this process, such as the gate's scope around `just test`.
*/
function strays(own?: string): Map<string, string> {
const self = readFileSync("/proc/self/cgroup", "utf8")
.split("\n")
.find((line) => line.startsWith("0::"))!
.slice(3);
const show = execFileSync("systemctl", ["--user", "show", "--property=Id,ControlGroup", "quest-*.scope"], {
Comment thread
kixelated marked this conversation as resolved.
Resolved
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
const found = new Map<string, string>();
for (const entry of show.split("\n\n")) {
const unit = /^Id=(.+)$/m.exec(entry)?.[1];
const group = /^ControlGroup=(.+)$/m.exec(entry)?.[1];
if (!unit || !group || self === group || self.startsWith(`${group}/`)) continue;
const path = join("/sys/fs/cgroup", group);
const runs = members(path).map(runOf);
if (runs.some((run) => run !== undefined && (run === own || dead(run)))) found.set(unit, path);
}
return found;
}

/** Stop every scope a run that exited left running, and those of run `own`; resolves once each is empty. */
export async function sweep(own?: string): Promise<void> {
const stops = await Promise.allSettled([...strays(own).keys()].map((unit) => systemdScopes.stop(unit)));
const failed = stops.flatMap((stop) => (stop.status === "rejected" ? [stop.reason] : []));
if (failed.length) throw new AggregateError(failed, `cannot stop ${failed.length} test scopes`);
}

/** Whether a user manager answers; without one, no test can start a scope, and none was left. */
function managed(): boolean {
try {
execFileSync("systemctl", ["--user", "show", "--property=Version", "--value"], { stdio: "ignore" });
return true;
} catch {
return false;
}
}

/** Vitest's global setup: marks the run, clears what dead runs left, and stops this run's scopes when it ends. */
export default async function setup(): Promise<(() => Promise<void>) | undefined> {
if (process.platform !== "linux") return undefined;
const own = `${process.pid}:${startTime(process.pid)}`;
// Set before the workers start, so they and everything they spawn inherit it.
process.env[RUN] = own;
// Such as in a container: the tests that start scopes fail there on their own, and the rest still run.
if (!managed()) return undefined;
// A run killed outright, such as by SIGKILL, never reached its teardown.
await sweep();
// A signalled run never reaches its teardown either.
for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"] as const) {
process.once(signal, () => {
// One scope that cannot be killed must not spare the rest, nor keep the run alive.
try {
for (const [unit, group] of strays(own))
try {
kill(group);
} catch (error) {
process.stderr.write(`cannot kill test scope ${unit}: ${error}\n`);
}
} catch (error) {
process.stderr.write(`cannot find the test scopes: ${error}\n`);
}
// Raised again once this listener is gone, the signal ends the run as it would have without
// it: signal-exit, which Vite's bundler loads, re-raises a signal only when it listens alone.
setImmediate(() => process.kill(process.pid, signal));
});
}
return () => sweep(own);
}
1 change: 1 addition & 0 deletions vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export default defineConfig({
// A test's CPU budget (test/budget.ts) bounds how slow it may be; this wall-clock
// limit only catches a test awaiting what never comes, so a loaded host never reaches it.
setupFiles: ["test/budget.ts"],
globalSetup: ["test/scopes.ts"],
testTimeout: 5 * 60_000,
hookTimeout: 5 * 60_000,
}),
Expand Down
Loading