From 0e2f447b214711f5cb42245ae1521fcce6dfda27 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 22:25:39 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m0/gate-memory.md Quest-Claim: 768cd5ac4daed53a From 8f3b10e4ce02df5efaabd24936acbe7988dbb572 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 23:41:17 -0800 Subject: [PATCH 2/6] feat(gate): hold each gated command to a memory ceiling its gate raises while the host has room Checkpoint: scope ceilings, ledger accounting, and tests; docs and the quest file follow. Co-Authored-By: Claude Fable 5.2 --- src/cli/gate.ts | 227 ++++++++++++++++++++++++++++++++++++++----- src/cli/supervise.ts | 82 +++++++++++++++- test/gate.test.ts | 193 +++++++++++++++++++++++++++++++++++- 3 files changed, 473 insertions(+), 29 deletions(-) diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 2fdaa6d6..d769278b 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -3,14 +3,24 @@ // the command's descendants until they stop, and holds nothing else while it runs. import { spawn, spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { linkSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + existsSync, + linkSync, + lstatSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { availableParallelism, constants, cpus, hostname, loadavg, totalmem, userInfo } from "node:os"; import { dirname, join, relative, resolve, sep } from "node:path"; import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; import { stubbed } from "./job"; -import { questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; +import { type Ceiling, capped, limit, questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; /** * One admitted command: a unique id, the wall-clock time it was admitted, and the process ID of @@ -18,7 +28,9 @@ import { questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; * (see `processStart`), so a reused process ID holds nothing. `queued` marks a caller still * waiting for its turn instead, as of `at`: it holds a place in the queue, the ledger's order of * queued entries, and no slot. `cores` is the most CPUs the command kept busy over any - * SETTLE_MS of the last RECENT_MS, where its gate measures it (see `measure`). + * SETTLE_MS of the last RECENT_MS, where its gate measures it (see `measure`). `memory` is the + * ceiling its scope holds its memory to, in bytes, where the gate caps it, and `held` what its + * command held of that when last sampled (see `ration`). */ export interface Admission { id: string; @@ -27,6 +39,16 @@ export interface Admission { queued?: boolean; start?: string; cores?: number; + memory?: number; + held?: number; +} + +/** What the processes in a scope have used. */ +export interface Usage { + /** CPU microseconds. */ + cpu: number; + /** Bytes of memory the kernel cannot reclaim from them; undefined where the scope has no memory accounting. */ + memory?: number; } /** One version of the host's admission ledger and queue. Versions only grow; 0 is the empty ledger. */ @@ -53,8 +75,10 @@ export interface Host { */ publish(base: Ledger, admissions: Admission[]): boolean; sleep(ms: number, signal: AbortSignal): Promise; - /** CPU microseconds the processes in systemd scope `unit` have used, or undefined while it does not exist. */ - usage(unit: string): number | undefined; + /** What the processes in systemd scope `unit` have used, or undefined while it does not exist. */ + usage(unit: string): Usage | undefined; + /** Move the memory ceiling of systemd scope `unit`. */ + cap(unit: string, memory: Ceiling): void; } export interface Limits { @@ -62,6 +86,31 @@ export interface Limits { load: number; /** The least available memory, in bytes, that still admits a command. */ memory: number; + /** + * The memory ceiling an admitted command's scope starts with, in bytes, and the least its gate + * raises it by (see `ration`); undefined where no scope caps the command. + */ + ceiling?: number; +} + +/** A ceiling starts at this fraction of one command's memory, so what a small command never uses stays free. */ +const CEILING_PARTS = 8; + +/** How often a gate compares its command's memory with its ceiling. */ +export const CEILING_MS = 1000; + +/** + * The scope limits for a ceiling of `share` bytes that grows by `step`: the kernel slows the + * command a little below the ceiling, so it is held there briefly while its gate looks for room + * to raise both, and is killed only once it fills the rest. + */ +export function bounds(share: number, step: number): Ceiling { + return { high: share - Math.floor(Math.min(step / 4, share / 16)), max: share }; +} + +/** What a running command may still take of its ceiling: memory promised to it, so not free for another. */ +function unused(admission: Admission): number { + return Math.max(0, (admission.memory ?? 0) - (admission.held ?? 0)); } export type Waiting = "load" | "memory" | "queue"; @@ -122,7 +171,7 @@ export function hostLimits( ): Limits { return { load: processors, - memory: Math.min(COMMAND_MEMORY, memory / 4), + memory: Math.floor(Math.min(COMMAND_MEMORY, memory / 4)), }; } @@ -257,7 +306,9 @@ function parse(text: string): Admission[] { (entry.pid === undefined || (Number.isSafeInteger(entry.pid) && entry.pid > 0)) && (entry.queued === undefined || entry.queued === true) && (entry.start === undefined || typeof entry.start === "string") && - (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)), + (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)) && + (entry.memory === undefined || (Number.isSafeInteger(entry.memory) && entry.memory >= 0)) && + (entry.held === undefined || (Number.isSafeInteger(entry.held) && entry.held >= 0)), ); return valid ? (value as Admission[]) : []; } @@ -399,17 +450,30 @@ export function fileHost(directory: string, now = Date.now): Host { if (group === undefined) return undefined; groups.set(unit, group); let stat: string; + let memory: number | undefined; try { stat = readFileSync(join(group, "cpu.stat"), "utf8"); + // File cache counts as the scope's memory, but the kernel reclaims it before it slows or + // kills the command. + if (existsSync(join(group, "memory.current"))) + memory = Math.max( + 0, + Number(readFileSync(join(group, "memory.current"), "utf8")) - + cgroupStat(group, "active_file") - + cgroupStat(group, "inactive_file"), + ); } catch (error) { // The scope ended, and systemd removed its cgroup. - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + if (["ENOENT", "ENODEV"].includes((error as NodeJS.ErrnoException).code ?? "")) return undefined; throw error; } const usage = stat.match(/^usage_usec (\d+)$/m)?.[1]; if (usage === undefined) throw new Error(`gate cannot read the CPU use of ${unit}`); - return Number(usage); + if (memory !== undefined && !Number.isFinite(memory)) + throw new Error(`gate cannot read the memory of ${unit}`); + return { cpu: Number(usage), memory }; }, + cap: limit, }; } @@ -506,7 +570,10 @@ export async function admit( const ahead = kept.slice(0, Math.max(position, 0)).filter((admission) => admission.queued).length; // An admission takes the place of this caller's entry in the queue. const admitted = (at: number) => - host.publish(ledger, [...kept.filter((admission) => admission.id !== place), { id, at, ...self }]); + host.publish(ledger, [ + ...kept.filter((admission) => admission.id !== place), + { id, at, ...self, ...(limits.ceiling === undefined ? {} : { memory: limits.ceiling }) }, + ]); // Recorded like any admission, so it still counts against the others' headroom. if (now >= deadline) return admitted(now); // Priority ignores load, whether or not a gated command runs; memory still applies. @@ -516,7 +583,9 @@ export async function admit( : running.size > 0 ? Math.max(host.load(), used) : Math.min(host.load(), limits.load); - const memory = host.memory(); + // Running commands' ceilings must fit the host together, so what each may still take of its + // own is not free, however long it has run. + const memory = host.memory() - [...running].reduce((sum, admission) => sum + unused(admission), 0); let wait = SETTLE_MS; if (load > limits.load || used >= limits.load) waiting?.("load"); else if (memory < limits.memory) waiting?.("memory"); @@ -524,7 +593,13 @@ export async function admit( // Every caller queued ahead takes a slot before this one, so it starts first only while // the host has room for them all. const taken = recent.length + ahead; - const headroom = load + taken < limits.load && memory - taken * limits.memory >= limits.memory; + // A recent command has yet to show its memory, so it counts as one command's, less what + // its ceiling already took above. + const lagging = recent.reduce( + (sum, admission) => sum + Math.max(0, limits.memory - (running.has(admission) ? unused(admission) : 0)), + ahead * limits.memory, + ); + const headroom = load + taken < limits.load && memory - lagging >= limits.memory; if (!headroom && ahead > 0) waiting?.("queue"); else { const last = Math.max(-Infinity, ...recent.map((admission) => admission.at)); @@ -578,7 +653,7 @@ export async function measure(host: Host, admission: { id: string; unit: string if (signal.aborted) return; throw error; } - const usage = host.usage(admission.unit); + const usage = host.usage(admission.unit)?.cpu; const at = host.now(); if (usage !== undefined) { // A clock stepped backward yields no rate for its interval. @@ -596,6 +671,73 @@ export async function measure(host: Host, admission: { id: string; unit: string } } +/** + * Until `signal` aborts, keep the memory ceiling of the admitted command in systemd scope `unit` + * ahead of what it holds, while the host has room. Every CEILING_MS, the admission's `held` is set + * to what the command holds, rounded down to half a `step`, so other callers count the rest of its + * ceiling as promised. Once the command is within half a `step` of its ceiling, its gate raises the + * ceiling by a quarter of it, or by `step` if that is more or all that fits, taken from the + * host's available memory less what the other running commands may still take of theirs. Ceilings + * therefore never promise more than the host has, and never shrink. + * + * With no room, the ceiling stays: the kernel slows the command there and kills it once it fills + * the rest (see `bounds`), so the command that overran stops, not another. `throttled` is called + * with the ceiling each time the command starts being held at one. + */ +export async function ration( + host: Host, + admission: { id: string; unit: string; step: number }, + signal: AbortSignal, + throttled?: (ceiling: number) => void, +) { + const { id, unit, step } = admission; + let share = step; + let slowed = false; + for (;;) { + try { + await host.sleep(CEILING_MS, signal); + } catch (error) { + if (signal.aborted) return; + throw error; + } + const memory = host.usage(unit)?.memory; + if (memory === undefined) continue; + const held = memory - (memory % Math.ceil(step / 2)); + let grant = 0; + for (let ledger = host.ledger(); ; ledger = host.ledger()) { + const mine = ledger.admissions.find((entry) => entry.id === id); + if (mine === undefined) throw new Error(`gate lost its admission ${id} from the ledger`); + grant = 0; + if (memory > share - step / 2) { + const promised = ledger.admissions + .filter((entry) => entry !== mine && !entry.queued && host.running(entry)) + .reduce((sum, entry) => sum + unused(entry), 0); + const room = host.memory() - promised; + const wanted = Math.max(step, Math.floor(share / 4)); + grant = room >= wanted ? wanted : room >= step ? step : 0; + } + if (grant === 0 && mine.held === held) break; + const updated = { ...mine, memory: share + grant, held }; + if ( + host.publish( + ledger, + ledger.admissions.map((entry) => (entry === mine ? updated : entry)), + ) + ) + break; + } + if (grant > 0) { + // Recorded first, so no other caller counts this memory as free while the scope takes it. + share += grant; + host.cap(unit, bounds(share, step)); + slowed = false; + } else if (memory >= bounds(share, step).high) { + if (!slowed) throttled?.(share); + slowed = true; + } else slowed = false; + } +} + /** Marks an admitted command, so nested gated calls inside it run at once. */ const ADMITTED = "QUEST_GATE_ADMITTED"; @@ -657,20 +799,27 @@ export const inside: Owner = plain(true); * owner's kill stops the gate's scope too, and the gate forwards that owner's signals to it. */ const background = - (unit: string, enclosing: string | null): Owner => + (unit: string, enclosing: string | null, memory?: Ceiling): Owner => (command, env, cancel) => systemdScopes.run(unit, process.cwd(), command, { env, cancel, within: enclosing, slice: "background.slice", + memory, }); -/** Run the command in background scope `unit`, a new name, outside any Quest scope (see `background`). */ -export const scope = (unit: string): Owner => background(unit, null); +/** + * Run the command in background scope `unit`, a new name, outside any Quest scope (see + * `background`), held to `memory` when given. + */ +export const scope = (unit: string, memory?: Ceiling): Owner => background(unit, null, memory); -/** Run the command in background scope `unit`, a new name, bound to the Quest scope `enclosing` (see `background`). */ -export const bound = (unit: string, enclosing: string): Owner => background(unit, enclosing); +/** + * Run the command in background scope `unit`, a new name, bound to the Quest scope `enclosing` (see + * `background`), held to `memory` when given. + */ +export const bound = (unit: string, enclosing: string, memory?: Ceiling): Owner => background(unit, enclosing, memory); /** * On Windows, run the command under a stub in a kill-on-close Job Object that forbids breakaway @@ -764,21 +913,37 @@ const TOKEN = 3; * call inside a Quest scope runs its command in a background scope bound to it. Linux owns descendants through a systemd scope, and settles for a process group, * like macOS, without a systemd user manager. Windows owns them through a Job Object. A systemd * scope also lets the gate measure its command's CPU use (see `measure`), so `unit` names it. + * Where the user manager can limit memory, the scope starts with a ceiling of `ceiling` bytes, a + * fraction of `memory`, one command's, which the gate then raises (see `ration`). A nested call's + * command stays in the admitted command's cgroup, under its ceiling; every scope the gate starts + * has an admission, and so a ceiling, of its own. */ -export function owner(env = process.env, cgroup = "/proc/self/cgroup"): { run: Owner; unit?: string } { +export function owner( + env = process.env, + cgroup = "/proc/self/cgroup", + memory = hostLimits().memory, +): { run: Owner; unit?: string; ceiling?: number } { // A Quest scope around this process holds the command too, since it shares the cgroup or an ancestor. const enclosing = process.platform === "linux" ? questScope(cgroup) : undefined; if (env[ADMITTED] === "1") return { run: env[GROUP] === "1" || enclosing ? inside : direct }; if (process.platform === "win32") return { run: job }; const unit = `quest-gate-${randomUUID()}.scope`; - if (enclosing) return { run: bound(unit, enclosing.unit), unit }; + const held = () => { + const ceiling = Math.floor(memory / CEILING_PARTS); + return capped() && ceiling > 0 ? { ceiling, limits: bounds(ceiling, ceiling) } : {}; + }; + if (enclosing) { + const { ceiling, limits } = held(); + return { run: bound(unit, enclosing.unit, limits), unit, ceiling }; + } if (process.platform !== "linux") return { run: group }; try { systemdScopes.check(); - return { run: scope(unit), unit }; } catch { return { run: group }; } + const { ceiling, limits } = held(); + return { run: scope(unit, limits), unit, ceiling }; } const notices: Record = { @@ -795,7 +960,8 @@ const notices: Record = { export async function gate(command: string[], options: { quick?: boolean } = {}): Promise { const cancel = interrupts(); try { - const { run, unit } = owner(); + const limits = hostLimits(); + const { run, unit, ceiling } = owner(process.env, undefined, limits.memory); let admitted: { host: Host; id: string } | undefined; // A nested invocation executes inside the already admitted command. if (process.env[ADMITTED] !== "1") { @@ -811,7 +977,7 @@ export async function gate(command: string[], options: { quick?: boolean } = {}) quick: options.quick, }; const host = fileHost(claimDirectory()); - admitted = { host, id: await admit(hostLimits(), host, cancel.signal, notice, precedence) }; + admitted = { host, id: await admit({ ...limits, ceiling }, host, cancel.signal, notice, precedence) }; } catch (error) { if (cancel.signal.aborted) return cancelled(cancel.signal); throw error; @@ -819,9 +985,20 @@ export async function gate(command: string[], options: { quick?: boolean } = {}) } const stop = new AbortController(); // Held as a value until the command ends, so a failure never goes unhandled meanwhile. + const { host, id } = admitted ?? {}; + const throttled = (share: number) => + process.stderr.write( + `quest gate: the command is held at its memory ceiling of ${(share / 2 ** 30).toFixed(2)} GiB ` + + "until the host has room to raise it\n", + ); const measuring = - admitted !== undefined && unit !== undefined - ? measure(admitted.host, { id: admitted.id, unit }, stop.signal).then( + host !== undefined && id !== undefined && unit !== undefined + ? Promise.all([ + measure(host, { id, unit }, stop.signal), + ceiling === undefined + ? undefined + : ration(host, { id, unit, step: ceiling }, stop.signal, throttled), + ]).then( () => undefined, (error: unknown) => ({ error }), ) diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index feac04a2..4469908d 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -73,6 +73,17 @@ export interface RunOptions { within?: string | null; /** The slice the scope runs in; the user manager's default, `app.slice`, when omitted. */ slice?: string; + /** The memory ceiling the scope starts with; none when omitted. */ + memory?: Ceiling; +} + +/** + * The most memory a scope's processes may hold together, in bytes, with no swap beyond it. Above + * `high` the kernel slows them and reclaims their file cache; at `max` it kills them all. + */ +export interface Ceiling { + high: number; + max: number; } export interface Scopes { @@ -87,7 +98,8 @@ export interface Scopes { * cancellation signalled the whole scope, once the scope empties or the run is killed, so each * process can finish its own cleanup. Resolves with the command's exit status, or the cancelled * status once asked to stop, only once the scope is empty. Rejects with `Retained` when that - * cannot be confirmed, and otherwise only when the command never started. + * cannot be confirmed, with `Exhausted` when the kernel killed the scope at its memory ceiling, + * and otherwise only when the command never started. */ run(unit: string, cwd: string, command: string[], options?: RunOptions): Promise; } @@ -95,6 +107,12 @@ export interface Scopes { /** The command may have started and its scope may still hold processes, so its run stays recorded. */ export class Retained extends Error {} +/** The kernel killed every process in the scope, since they held its whole memory ceiling. */ +export class Exhausted extends Error {} + +/** The systemd properties that hold a scope to `memory`. */ +const ceiling = ({ high, max }: Ceiling) => [`MemoryHigh=${high}`, `MemoryMax=${max}`]; + export const UNSUPPORTED = "quest worktree run needs Linux 5.14 or later with a systemd user manager (systemd 254 or later) and unified cgroup v2"; @@ -252,6 +270,33 @@ async function stopBound(unit: string): Promise { for (const bound of boundTo(unit)) await systemdScopes.stop(bound); } +/** Whether the user manager may limit a scope's memory: the memory controller is delegated to it. */ +export function capped(): boolean { + try { + const manager = execFileSync("systemctl", ["--user", "show", "--property=ControlGroup", "--value"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + return readFileSync(join("/sys/fs/cgroup", manager, "cgroup.controllers"), "utf8") + .trim() + .split(" ") + .includes("memory"); + } catch { + return false; + } +} + +/** Move a running scope's memory ceiling. A scope that already ended has none to move. */ +export function limit(unit: string, memory: Ceiling): void { + try { + execFileSync("systemctl", ["--user", "set-property", "--runtime", unit, ...ceiling(memory)], { + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + if (scopeGroup(unit) !== undefined) throw error; + } +} + export const systemdScopes: Scopes = { check() { if (process.platform !== "linux" || !killable()) throw new Error(UNSUPPORTED); @@ -289,6 +334,13 @@ export const systemdScopes: Scopes = { // Once the enclosing owner kills everything it holds, systemd kills this scope too. const enclosing = options.within === undefined ? questScope()?.unit : options.within; const bound = enclosing ? [`--property=BindsTo=${enclosing}`, "--property=KillSignal=SIGKILL"] : []; + // Swap would let the scope hold more than its ceiling, and one process killed there would + // leave the rest of the command running without it. + const capped = options.memory + ? [...ceiling(options.memory), "MemorySwapMax=0", "OOMPolicy=kill"].map( + (property) => `--property=${property}`, + ) + : []; return contain( [ "systemd-run", @@ -298,6 +350,7 @@ export const systemdScopes: Scopes = { "--collect", `--unit=${unit}`, ...bound, + ...capped, ...(options.slice ? [`--slice=${options.slice}`] : []), "--expand-environment=no", "--", @@ -367,6 +420,20 @@ function contain( let admitted = false; let interrupted = 0; let failure: Error | undefined; + // The scope's ceiling in bytes, once the kernel killed a process there for lack of memory. + let exhausted: string | undefined; + let events: ReturnType | undefined; + const starved = () => { + if (exhausted !== undefined) return; + try { + if (/^oom_kill [1-9]/m.test(readFileSync(join(group!, "memory.events"), "utf8"))) + exhausted = readFileSync(join(group!, "memory.max"), "utf8").trim(); + } catch (error) { + // The scope ended, and systemd removed its cgroup. + if (!["ENOENT", "ENODEV"].includes((error as NodeJS.ErrnoException).code ?? "")) + failure = error as Error; + } + }; // Set once every process in the cgroup was asked to stop; aborted once they no longer get time to. let signalled = false; const drain = new AbortController(); @@ -418,6 +485,12 @@ function contain( return; } group = join("/sys/fs/cgroup", path); + if (options.memory) { + // The kernel notifies memory.events as it kills, while the cgroup still holds the dying + // processes; systemd removes the cgroup, and the count with it, once they are gone. + events = watch(join(group, "memory.events"), starved); + events.on("error", () => {}); + } if (interrupted) return; admitted = true; admission.end("admitted\n"); @@ -435,6 +508,8 @@ function contain( try { // Descendants that outlive the command are orphans of this run: stop them. if (group) { + if (events) starved(); + events?.close(); try { // Every process got the signal, so let each finish its own cleanup, even one that // outlives the wrappers that exited first, until a second signal kills them. @@ -449,6 +524,11 @@ function contain( } } if (failure) throw failure; + if (exhausted !== undefined && !interrupted) + throw new Exhausted( + `${describe.name} ran out of memory: the kernel killed its command at a ceiling of ` + + `${(Number(exhausted) / 2 ** 30).toFixed(2)} GiB, which its gate raises only while the host has free memory`, + ); if (!admitted && !interrupted) throw new Error( `${describe.unsupported}: ${describe.launcher} exited with ${code ?? signal} before the command started`, diff --git a/test/gate.test.ts b/test/gate.test.ts index 13726f79..ee5a81ce 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -20,6 +20,8 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { admit, alive, + bounds, + ration, COMMAND_MEMORY, claimDirectory, direct, @@ -43,7 +45,7 @@ import { } from "../src/cli/gate"; import { forwards, foregroundStat, statProcess } from "../src/cli/interrupt"; import { gateBoundary, main } from "../src/cli/main"; -import { alive as ownerAlive, questScope } from "../src/cli/supervise"; +import { capped, Exhausted, alive as ownerAlive, questScope } from "../src/cli/supervise"; import { created, onTerminal, run, signalled, tempDir } from "./fixture"; // Linux reports a /proc read that races a process's exit as ESRCH rather than ENOENT, so fail @@ -244,7 +246,7 @@ async function measured(busy: (number | undefined)[]) { host.publish(host.ledger(), [{ id: "build", at: 0, ...me }]); let usage: number | undefined; let total = 0; - host.usage = () => usage; + host.usage = () => (usage === undefined ? undefined : { cpu: usage }); const published: string[] = []; const publish = host.publish; host.publish = (base, admissions) => { @@ -281,6 +283,117 @@ test("a gate records its command's burst before the first sample", async () => { expect(published).toEqual([`4 at ${SETTLE_MS}`, `0 at ${SETTLE_MS + RECENT_MS}`]); }); +test("what running commands may still take of their ceilings is not free, however long they have run", async () => { + const events: string[] = []; + const state = { now: 0, load: 0, memory: 6 * GiB }; + const host = testHost(tempDir(), state, events); + // Two builds admitted long ago hold 1 GiB of the 6 GiB ceilings between them. + const builds = ["a", "b"].map((id) => ({ id, at: -10 * RECENT_MS, ...me, memory: 3 * GiB, held: GiB / 2 })); + expect(host.publish(host.ledger(), builds)).toBe(true); + const sleep = host.sleep; + host.sleep = async (ms, signal) => { + await sleep(ms, signal); + // One of them ends, so its ceiling is free again. + host.publish(host.ledger(), [builds[0]]); + }; + const limits = { load: 4, memory: 2 * GiB, ceiling: GiB / 4 }; + await admit(limits, host, signal, (reason) => events.push(reason)); + // 6 GiB less the 5 GiB promised leaves no command's memory; less 2.5 GiB does. + expect(events).toEqual(["memory", `sleep ${SETTLE_MS}`]); + // The new command starts with a ceiling of its own. + expect(host.ledger().admissions).toEqual([ + builds[0], + { id: expect.any(String), at: SETTLE_MS, ...me, memory: GiB / 4 }, + ]); + // It is recent, so it still counts as one command's memory, not as its small ceiling alone: + // 6 - 2.5 - 0.25 GiB has room for one more only after the spacing. + events.length = 0; + host.sleep = sleep; + await admit(limits, host, signal, (reason) => events.push(reason)); + expect(events).toEqual([`sleep ${SETTLE_MS}`]); +}); + +/** + * Run `ration` for a command with a 1 GiB ceiling that grows by as much, over `samples`: what the + * command holds and what the host has available at each CEILING_MS. Returns each ceiling the scope + * was moved to, each one it was held at, and its admission. + */ +async function rationed(samples: { held: number; available: number }[], others: Admission[] = []) { + const state = { now: 0, load: 0, memory: Infinity }; + const host = testHost(tempDir(), state); + host.publish(host.ledger(), [{ id: "build", at: 0, ...me, memory: GiB }, ...others]); + let held = 0; + host.usage = () => ({ cpu: 0, memory: held }); + const caps: number[] = []; + host.cap = (_unit, { high, max }) => { + expect(high).toBeLessThan(max); + caps.push(max / GiB); + }; + const stop = new AbortController(); + host.sleep = async () => { + const sample = samples.shift(); + if (sample === undefined) { + stop.abort(); + throw stop.signal.reason; + } + held = sample.held * GiB; + state.memory = sample.available * GiB; + }; + const throttled: number[] = []; + await ration(host, { id: "build", unit: "quest-gate-build.scope", step: GiB }, stop.signal, (ceiling) => + throttled.push(ceiling / GiB), + ); + return { caps, throttled, mine: host.ledger().admissions[0] }; +} + +test("a gate raises its command's ceiling ahead of what it holds while the host has room", async () => { + const { caps, throttled, mine } = await rationed([ + { held: 0.25, available: 8 }, + // Within half a step of the ceiling: one more step. + { held: 0.75, available: 8 }, + { held: 1.25, available: 8 }, + { held: 1.9, available: 8 }, + { held: 2.75, available: 8 }, + { held: 3.8, available: 8 }, + // Past 4 GiB, a quarter of the ceiling is more than a step. + { held: 4.9, available: 8 }, + // It never shrinks. + { held: 0.5, available: 8 }, + ]); + expect(caps).toEqual([2, 3, 4, 5, 6.25]); + expect(throttled).toEqual([]); + expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 6.25 * GiB, held: GiB / 2 }); +}); + +test("with no room, a ceiling stays until other commands free memory", async () => { + // Another command may still take 3 GiB of its ceiling, which is not this one's to grow into. + const other = { id: "other", at: 0, ...me, memory: 4 * GiB, held: GiB }; + const { caps, throttled, mine } = await rationed( + [ + { held: 0.75, available: 3.5 }, + // Held where the kernel slows it: reported once, not at every sample. + { held: 0.95, available: 3.5 }, + { held: 0.96, available: 3.5 }, + // A step now fits, though a queued caller's place and a finished command hold nothing. + { held: 0.96, available: 4 }, + { held: 1.99, available: 3.9 }, + ], + [ + other, + { id: "queued", at: 0, ...me, queued: true }, + { id: "gone", at: 0, pid: 2 ** 22 + 1, memory: 64 * GiB }, + ], + ); + expect(caps).toEqual([2]); + expect(throttled).toEqual([1, 2]); + expect(mine).toMatchObject({ memory: 2 * GiB, held: 1.5 * GiB }); +}); + +test("a ceiling slows its command just below where it is killed", () => { + expect(bounds(GiB, GiB)).toEqual({ high: GiB - GiB / 16, max: GiB }); + expect(bounds(8 * GiB, GiB)).toEqual({ high: 8 * GiB - GiB / 4, max: 8 * GiB }); +}); + test("a priority caller waits for memory, never for load, even while a gated command runs", async () => { const directory = tempDir(); const state = { now: 0, load: 99, memory: 0 }; @@ -1095,11 +1208,85 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { expect(host.usage(unit)).toBeUndefined(); const spin = ["/bin/sh", "-c", `while [ ! -e ${quote(done)} ]; do :; done`]; const result = scope(unit)(spin, process.env, cancellation().cancel); - await until(() => (host.usage(unit) ?? 0) > 0); + await until(() => (host.usage(unit)?.cpu ?? 0) > 0); writeFileSync(done, ""); expect(await result).toBe(0); await until(() => host.usage(unit) === undefined); }); + + const MiB = 2 ** 20; + /** Hold `megabytes` of memory the kernel cannot reclaim, a megabyte at a time, then `then`. */ + const hog = (megabytes: number, then: string) => [ + process.execPath, + "-e", + `const held = []; for (let i = 0; i < ${megabytes}; i++) held.push(Buffer.alloc(${MiB}, 1)); ${then}`, + ]; + + test.runIf(capped())("holds its command to a memory ceiling the gate can raise", async () => { + const host = fileHost(tempDir()); + const unit = `quest-gate-${randomUUID()}.scope`; + const dir = tempDir(); + const report = join(dir, "limits"); + const raised = join(dir, "raised"); + const wait = `until [ -e ${quote(raised)} ]; do sleep 0.05; done`; + const limits = `cd /sys/fs/cgroup$(cut -d: -f3 /proc/self/cgroup) && cat memory.high memory.max memory.swap.max memory.oom.group`; + const result = scope(unit, { high: 192 * MiB, max: 256 * MiB })( + ["/bin/sh", "-c", `${limits} > ${quote(report)}; ${wait}; ${limits} >> ${quote(report)}`], + process.env, + cancellation().cancel, + ); + await until(() => existsSync(report) && readFileSync(report, "utf8").split("\n").length > 4); + expect(host.usage(unit)?.memory).toBeLessThan(192 * MiB); + host.cap(unit, { high: 448 * MiB, max: 512 * MiB }); + writeFileSync(raised, ""); + expect(await result).toBe(0); + expect(readFileSync(report, "utf8").trim().split("\n").map(Number)).toEqual([ + ...[192 * MiB, 256 * MiB, 0, 1], + ...[448 * MiB, 512 * MiB, 0, 1], + ]); + // A scope that already ended has no ceiling to move. + host.cap(unit, { high: 448 * MiB, max: 512 * MiB }); + }); + + test.runIf(capped())( + "a command that fills its ceiling is killed whole, and nothing outside its scope", + async () => { + const dir = tempDir(); + const pid = join(dir, "pid"); + const ready = join(dir, "ready"); + // A neighbour in a scope of its own holds memory beside it, and a wrapper waits on the hog. + const beside = cancellation(); + const neighbour = scoped( + hog(64, `require("node:fs").writeFileSync(${quote(ready)}, ""); setInterval(() => {}, 1000)`), + process.env, + beside.cancel, + ); + const stop = cancellation(); + await until(() => existsSync(ready)); + const run = scope(`quest-gate-${randomUUID()}.scope`, { high: 255 * MiB, max: 256 * MiB })( + [ + "/bin/sh", + "-c", + `sleep 1000 & echo $! > ${quote(pid)}; ${hog(1024, "").map(quote).join(" ")}; exit 0`, + ], + process.env, + stop.cancel, + ); + await expect(run).rejects.toThrow( + /ran out of memory: the kernel killed its command at a ceiling of 0\.25 GiB/, + ); + await expect(run).rejects.toBeInstanceOf(Exhausted); + // The wrapper and its other child died with the hog, and the scope beside it still runs. + await until(() => stopped(Number(readFileSync(pid, "utf8")))); + const state = await Promise.race([ + neighbour, + new Promise((resolve) => setTimeout(resolve, 100, "running")), + ]); + expect(state).toBe("running"); + beside.kill.abort(); + await neighbour; + }, + ); }); /** From a743cee1d1e5c550e2550d1a2310c6e656bbfe74 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 00:56:07 -0800 Subject: [PATCH 3/6] gate: keep a reserve free, lower a ceiling its command left, and read an out-of-memory kill from systemd Co-Authored-By: Claude Fable 5.2 --- src/cli/gate.ts | 71 ++++++++++++++++---------- src/cli/supervise.ts | 49 +++++++++--------- test/gate.test.ts | 118 ++++++++++++++++++++++++------------------- 3 files changed, 132 insertions(+), 106 deletions(-) diff --git a/src/cli/gate.ts b/src/cli/gate.ts index d769278b..1db92644 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -15,7 +15,7 @@ import { writeFileSync, } from "node:fs"; import { availableParallelism, constants, cpus, hostname, loadavg, totalmem, userInfo } from "node:os"; -import { dirname, join, relative, resolve, sep } from "node:path"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; @@ -100,12 +100,12 @@ const CEILING_PARTS = 8; export const CEILING_MS = 1000; /** - * The scope limits for a ceiling of `share` bytes that grows by `step`: the kernel slows the - * command a little below the ceiling, so it is held there briefly while its gate looks for room - * to raise both, and is killed only once it fills the rest. + * The scope limits for a ceiling of `share` bytes: the kernel slows the command over the last + * eighth below it, ever harder the nearer it gets, so its gate has time to raise both while the + * host has room, and kills it only once it fills the rest. */ -export function bounds(share: number, step: number): Ceiling { - return { high: share - Math.floor(Math.min(step / 4, share / 16)), max: share }; +export function bounds(share: number): Ceiling { + return { high: share - Math.floor(share / 8), max: share }; } /** What a running command may still take of its ceiling: memory promised to it, so not free for another. */ @@ -186,7 +186,10 @@ function cgroupStat(group: string, key: string): number { } } -/** Count reclaimable host cache and the headroom of every enclosing memory cgroup. */ +/** A gate scope's cgroup directory name. */ +const GATE_SCOPE = /^quest-gate-[0-9a-f-]+\.scope$/; + +/** Count reclaimable host cache and the headroom of every enclosing memory cgroup, except a gate's own ceiling. */ export function linuxMemory(proc = "/proc", cgroups = "/sys/fs/cgroup"): { total: number; available: number } { const meminfo = readFileSync(join(proc, "meminfo"), "utf8"); const bytes = (key: string) => { @@ -218,7 +221,8 @@ export function linuxMemory(proc = "/proc", cgroups = "/sys/fs/cgroup"): { total const inside = relative(root, start); if (inside === ".." || inside.startsWith(`..${sep}`)) continue; for (let group = start; ; group = dirname(group)) { - for (const file of limits) { + // A gate's ceiling binds its own scope, not the scope of its own a command started here gets. + for (const file of GATE_SCOPE.test(basename(group)) ? [] : limits) { try { const limit = Number(readFileSync(join(group, file), "utf8").trim()); if (!Number.isFinite(limit) || limit <= 0) continue; // "max", or an unset v1 soft limit. @@ -675,10 +679,15 @@ export async function measure(host: Host, admission: { id: string; unit: string * Until `signal` aborts, keep the memory ceiling of the admitted command in systemd scope `unit` * ahead of what it holds, while the host has room. Every CEILING_MS, the admission's `held` is set * to what the command holds, rounded down to half a `step`, so other callers count the rest of its - * ceiling as promised. Once the command is within half a `step` of its ceiling, its gate raises the - * ceiling by a quarter of it, or by `step` if that is more or all that fits, taken from the - * host's available memory less what the other running commands may still take of theirs. Ceilings - * therefore never promise more than the host has, and never shrink. + * ceiling as promised: + * + * - Once the command holds three quarters of its ceiling, its gate raises the ceiling by a quarter + * of it, or by `step` if that is more or all that fits. The raise comes from the host's available + * memory, less what the other running commands may still take of their ceilings and less + * `reserve`, which stays free for everything outside the gate. Ceilings therefore never promise + * more than the host has. + * - Once it holds less than half, the ceiling falls to a `step` above what it holds, so a peak the + * command has left is free for others. * * With no room, the ceiling stays: the kernel slows the command there and kills it once it fills * the rest (see `bounds`), so the command that overran stops, not another. `throttled` is called @@ -686,11 +695,11 @@ export async function measure(host: Host, admission: { id: string; unit: string */ export async function ration( host: Host, - admission: { id: string; unit: string; step: number }, + admission: { id: string; unit: string; step: number; reserve: number }, signal: AbortSignal, throttled?: (ceiling: number) => void, ) { - const { id, unit, step } = admission; + const { id, unit, step, reserve } = admission; let share = step; let slowed = false; for (;;) { @@ -703,21 +712,28 @@ export async function ration( const memory = host.usage(unit)?.memory; if (memory === undefined) continue; const held = memory - (memory % Math.ceil(step / 2)); - let grant = 0; + const fitted = Math.max(step, Math.ceil(memory / step) * step + step); + // Lowered before it is recorded, and raised after, so the ledger never promises less than the scope allows. + if (memory < share / 2 && fitted < share) { + share = fitted; + host.cap(unit, bounds(share)); + } + let next = share; for (let ledger = host.ledger(); ; ledger = host.ledger()) { const mine = ledger.admissions.find((entry) => entry.id === id); if (mine === undefined) throw new Error(`gate lost its admission ${id} from the ledger`); - grant = 0; - if (memory > share - step / 2) { + next = share; + if (memory > share - share / 4) { const promised = ledger.admissions .filter((entry) => entry !== mine && !entry.queued && host.running(entry)) .reduce((sum, entry) => sum + unused(entry), 0); - const room = host.memory() - promised; + // This command's own unused ceiling is already promised to it. + const room = host.memory() - promised - reserve - Math.max(0, share - memory); const wanted = Math.max(step, Math.floor(share / 4)); - grant = room >= wanted ? wanted : room >= step ? step : 0; + next += room >= wanted ? wanted : room >= step ? step : 0; } - if (grant === 0 && mine.held === held) break; - const updated = { ...mine, memory: share + grant, held }; + if (mine.memory === next && mine.held === held) break; + const updated = { ...mine, memory: next, held }; if ( host.publish( ledger, @@ -726,12 +742,11 @@ export async function ration( ) break; } - if (grant > 0) { - // Recorded first, so no other caller counts this memory as free while the scope takes it. - share += grant; - host.cap(unit, bounds(share, step)); + if (next > share) { + share = next; + host.cap(unit, bounds(share)); slowed = false; - } else if (memory >= bounds(share, step).high) { + } else if (memory >= bounds(share).high) { if (!slowed) throttled?.(share); slowed = true; } else slowed = false; @@ -930,7 +945,7 @@ export function owner( const unit = `quest-gate-${randomUUID()}.scope`; const held = () => { const ceiling = Math.floor(memory / CEILING_PARTS); - return capped() && ceiling > 0 ? { ceiling, limits: bounds(ceiling, ceiling) } : {}; + return capped() && ceiling > 0 ? { ceiling, limits: bounds(ceiling) } : {}; }; if (enclosing) { const { ceiling, limits } = held(); @@ -997,7 +1012,7 @@ export async function gate(command: string[], options: { quick?: boolean } = {}) measure(host, { id, unit }, stop.signal), ceiling === undefined ? undefined - : ration(host, { id, unit, step: ceiling }, stop.signal, throttled), + : ration(host, { id, unit, step: ceiling, reserve: limits.memory }, stop.signal, throttled), ]).then( () => undefined, (error: unknown) => ({ error }), diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index 4469908d..c1f5eebc 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -297,6 +297,21 @@ export function limit(unit: string, memory: Ceiling): void { } } +/** + * Why systemd counts the ended scope as failed, such as `oom-kill`, forgetting the unit; undefined + * for one that ended normally. systemd records an out-of-memory kill before it handles this + * request, since it reads the kernel's notice of one ahead of its bus. + */ +function failed(unit: string): string | undefined { + const result = execFileSync("systemctl", ["--user", "show", "--property=Result", "--value", unit], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + if (result === "success") return undefined; + execFileSync("systemctl", ["--user", "reset-failed", unit], { stdio: ["ignore", "pipe", "pipe"] }); + return result; +} + export const systemdScopes: Scopes = { check() { if (process.platform !== "linux" || !killable()) throw new Error(UNSUPPORTED); @@ -347,7 +362,8 @@ export const systemdScopes: Scopes = { "--user", "--scope", "--quiet", - "--collect", + // A scope the kernel killed at its ceiling stays loaded as failed, so its run can ask why it ended. + ...(options.memory ? [] : ["--collect"]), `--unit=${unit}`, ...bound, ...capped, @@ -365,6 +381,7 @@ export const systemdScopes: Scopes = { unsupported: UNSUPPORTED, launcher: "systemd-run", emptied: () => stopBound(unit), + exhausted: () => options.memory !== undefined && failed(unit) === "oom-kill", bound: Boolean(enclosing), }, cwd, @@ -384,6 +401,8 @@ interface Containment { launcher: string; /** Stop what the cgroup's end leaves to systemd, once the cgroup is empty. */ emptied(): Promise; + /** Whether the kernel killed the cgroup's processes at its memory ceiling, asked once they are gone. */ + exhausted(): boolean; /** * Whether an enclosing Quest owner kills this cgroup at its own second signal, so a repeated * signal this process receives, which may be a copy of that owner's, leaves the kill to it. @@ -420,20 +439,6 @@ function contain( let admitted = false; let interrupted = 0; let failure: Error | undefined; - // The scope's ceiling in bytes, once the kernel killed a process there for lack of memory. - let exhausted: string | undefined; - let events: ReturnType | undefined; - const starved = () => { - if (exhausted !== undefined) return; - try { - if (/^oom_kill [1-9]/m.test(readFileSync(join(group!, "memory.events"), "utf8"))) - exhausted = readFileSync(join(group!, "memory.max"), "utf8").trim(); - } catch (error) { - // The scope ended, and systemd removed its cgroup. - if (!["ENOENT", "ENODEV"].includes((error as NodeJS.ErrnoException).code ?? "")) - failure = error as Error; - } - }; // Set once every process in the cgroup was asked to stop; aborted once they no longer get time to. let signalled = false; const drain = new AbortController(); @@ -485,12 +490,6 @@ function contain( return; } group = join("/sys/fs/cgroup", path); - if (options.memory) { - // The kernel notifies memory.events as it kills, while the cgroup still holds the dying - // processes; systemd removes the cgroup, and the count with it, once they are gone. - events = watch(join(group, "memory.events"), starved); - events.on("error", () => {}); - } if (interrupted) return; admitted = true; admission.end("admitted\n"); @@ -508,8 +507,6 @@ function contain( try { // Descendants that outlive the command are orphans of this run: stop them. if (group) { - if (events) starved(); - events?.close(); try { // Every process got the signal, so let each finish its own cleanup, even one that // outlives the wrappers that exited first, until a second signal kills them. @@ -524,10 +521,10 @@ function contain( } } if (failure) throw failure; - if (exhausted !== undefined && !interrupted) + if (code !== 0 && !interrupted && describe.exhausted()) throw new Exhausted( - `${describe.name} ran out of memory: the kernel killed its command at a ceiling of ` + - `${(Number(exhausted) / 2 ** 30).toFixed(2)} GiB, which its gate raises only while the host has free memory`, + `${describe.name} ran out of memory: the kernel killed its command at its memory ceiling, ` + + "which its gate raises only while the host has free memory", ); if (!admitted && !interrupted) throw new Error( diff --git a/test/gate.test.ts b/test/gate.test.ts index ee5a81ce..96937cd6 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -314,9 +314,10 @@ test("what running commands may still take of their ceilings is not free, howeve }); /** - * Run `ration` for a command with a 1 GiB ceiling that grows by as much, over `samples`: what the - * command holds and what the host has available at each CEILING_MS. Returns each ceiling the scope - * was moved to, each one it was held at, and its admission. + * Run `ration` for a command with a 1 GiB ceiling that moves by as much, on a host that keeps + * 1 GiB free, over `samples`: what the command holds and what the host has available at each + * CEILING_MS. Returns each ceiling the scope was moved to, each one it was held at, and its + * admission. */ async function rationed(samples: { held: number; available: number }[], others: Admission[] = []) { const state = { now: 0, load: 0, memory: Infinity }; @@ -326,7 +327,9 @@ async function rationed(samples: { held: number; available: number }[], others: host.usage = () => ({ cpu: 0, memory: held }); const caps: number[] = []; host.cap = (_unit, { high, max }) => { - expect(high).toBeLessThan(max); + expect(high).toBe(max - max / 8); + // The ledger already promises a raised ceiling, and still promises a lowered one. + expect(host.ledger().admissions[0].memory).toBeGreaterThanOrEqual(max); caps.push(max / GiB); }; const stop = new AbortController(); @@ -340,29 +343,34 @@ async function rationed(samples: { held: number; available: number }[], others: state.memory = sample.available * GiB; }; const throttled: number[] = []; - await ration(host, { id: "build", unit: "quest-gate-build.scope", step: GiB }, stop.signal, (ceiling) => - throttled.push(ceiling / GiB), + await ration( + host, + { id: "build", unit: "quest-gate-build.scope", step: GiB, reserve: GiB }, + stop.signal, + (ceiling) => throttled.push(ceiling / GiB), ); return { caps, throttled, mine: host.ledger().admissions[0] }; } -test("a gate raises its command's ceiling ahead of what it holds while the host has room", async () => { +test("a gate keeps its command's ceiling ahead of what it holds while the host has room", async () => { const { caps, throttled, mine } = await rationed([ - { held: 0.25, available: 8 }, - // Within half a step of the ceiling: one more step. - { held: 0.75, available: 8 }, - { held: 1.25, available: 8 }, - { held: 1.9, available: 8 }, - { held: 2.75, available: 8 }, - { held: 3.8, available: 8 }, + { held: 0.25, available: 9 }, + // Three quarters of the ceiling: one more step. + { held: 0.8, available: 9 }, + { held: 1.6, available: 9 }, + { held: 2.4, available: 9 }, + { held: 3.2, available: 9 }, // Past 4 GiB, a quarter of the ceiling is more than a step. - { held: 4.9, available: 8 }, - // It never shrinks. - { held: 0.5, available: 8 }, + { held: 4, available: 9 }, + // Half of it is no reason to move. + { held: 3.2, available: 9 }, + // Less is: a step above what it holds. + { held: 1.2, available: 9 }, + { held: 0.1, available: 9 }, ]); - expect(caps).toEqual([2, 3, 4, 5, 6.25]); + expect(caps).toEqual([2, 3, 4, 5, 6.25, 3, 2]); expect(throttled).toEqual([]); - expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 6.25 * GiB, held: GiB / 2 }); + expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 2 * GiB, held: 0 }); }); test("with no room, a ceiling stays until other commands free memory", async () => { @@ -370,13 +378,14 @@ test("with no room, a ceiling stays until other commands free memory", async () const other = { id: "other", at: 0, ...me, memory: 4 * GiB, held: GiB }; const { caps, throttled, mine } = await rationed( [ - { held: 0.75, available: 3.5 }, + // 4.7 GiB less the 3 GiB promised, the 1 GiB kept free, and its own 0.2 GiB left: no step. + { held: 0.8, available: 4.7 }, // Held where the kernel slows it: reported once, not at every sample. - { held: 0.95, available: 3.5 }, - { held: 0.96, available: 3.5 }, + { held: 0.9, available: 4.6 }, + { held: 0.95, available: 4.6 }, // A step now fits, though a queued caller's place and a finished command hold nothing. - { held: 0.96, available: 4 }, - { held: 1.99, available: 3.9 }, + { held: 0.95, available: 5.1 }, + { held: 1.9, available: 4.2 }, ], [ other, @@ -389,9 +398,8 @@ test("with no room, a ceiling stays until other commands free memory", async () expect(mine).toMatchObject({ memory: 2 * GiB, held: 1.5 * GiB }); }); -test("a ceiling slows its command just below where it is killed", () => { - expect(bounds(GiB, GiB)).toEqual({ high: GiB - GiB / 16, max: GiB }); - expect(bounds(8 * GiB, GiB)).toEqual({ high: 8 * GiB - GiB / 4, max: 8 * GiB }); +test("a ceiling slows its command over the last eighth before it is killed", () => { + expect(bounds(GiB)).toEqual({ high: GiB - GiB / 8, max: GiB }); }); test("a priority caller waits for memory, never for load, even while a gated command runs", async () => { @@ -789,6 +797,13 @@ test("Linux memory reads the container cgroup mounted as the root", () => { // A cgroup outside the namespace is not mounted, so nothing visible bounds it. writeFileSync(join(proc, "self/cgroup"), "0::/../sibling\n"); expect(linuxMemory(proc, cgroups)).toEqual({ total: 64 * GiB, available: 24 * GiB }); + // A gate's ceiling around the caller does not bound a command that gets a scope of its own. + const gated = "background.slice/quest-gate-0f18849e-1c60-4b6e-9d2a-7a1c2b3d4e5f.scope"; + mkdirSync(join(cgroups, gated), { recursive: true }); + writeFileSync(join(cgroups, gated, "memory.max"), String(GiB)); + writeFileSync(join(cgroups, gated, "memory.current"), String(GiB)); + writeFileSync(join(proc, "self/cgroup"), `0::/${gated}\n`); + expect(linuxMemory(proc, cgroups)).toEqual(limited); }); test("every caller on this host shares one ledger, whatever its environment or sessions", () => { @@ -1257,34 +1272,33 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { // A neighbour in a scope of its own holds memory beside it, and a wrapper waits on the hog. const beside = cancellation(); const neighbour = scoped( - hog(64, `require("node:fs").writeFileSync(${quote(ready)}, ""); setInterval(() => {}, 1000)`), + hog( + 64, + `require("node:fs").writeFileSync(${quote(ready)}, String(process.pid)); setInterval(() => {}, 1000)`, + ), process.env, beside.cancel, ); - const stop = cancellation(); - await until(() => existsSync(ready)); - const run = scope(`quest-gate-${randomUUID()}.scope`, { high: 255 * MiB, max: 256 * MiB })( - [ - "/bin/sh", - "-c", - `sleep 1000 & echo $! > ${quote(pid)}; ${hog(1024, "").map(quote).join(" ")}; exit 0`, - ], - process.env, - stop.cancel, - ); - await expect(run).rejects.toThrow( - /ran out of memory: the kernel killed its command at a ceiling of 0\.25 GiB/, - ); - await expect(run).rejects.toBeInstanceOf(Exhausted); - // The wrapper and its other child died with the hog, and the scope beside it still runs. - await until(() => stopped(Number(readFileSync(pid, "utf8")))); - const state = await Promise.race([ - neighbour, - new Promise((resolve) => setTimeout(resolve, 100, "running")), - ]); - expect(state).toBe("running"); - beside.kill.abort(); - await neighbour; + try { + await until(() => existsSync(ready)); + const run = scope(`quest-gate-${randomUUID()}.scope`, { high: 255 * MiB, max: 256 * MiB })( + [ + "/bin/sh", + "-c", + `sleep 1000 & echo $! > ${quote(pid)}; ${hog(1024, "").map(quote).join(" ")}; exit 0`, + ], + process.env, + cancellation().cancel, + ); + await expect(run).rejects.toThrow(/ran out of memory: the kernel killed its command/); + await expect(run).rejects.toBeInstanceOf(Exhausted); + // The wrapper and its other child died with the hog, and the scope beside it still runs. + await until(() => stopped(Number(readFileSync(pid, "utf8")))); + expect(stopped(Number(readFileSync(ready, "utf8")))).toBe(false); + } finally { + beside.kill.abort(); + await neighbour; + } }, ); }); From c9ec7a12dbfa0cd8c065f4ccdef369d79087884a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 01:32:25 -0800 Subject: [PATCH 4/6] gate: raise a ceiling by half, document ceilings, and finish quest/m0/gate-memory.md Co-Authored-By: Claude Fable 5.2 --- README.md | 23 +++++++++++++++ quest/m0/gate-memory.md | 62 ----------------------------------------- src/cli/gate.ts | 7 +++-- src/cli/main.ts | 4 ++- test/gate.test.ts | 9 +++--- 5 files changed, 34 insertions(+), 71 deletions(-) delete mode 100644 quest/m0/gate-memory.md diff --git a/README.md b/README.md index 4d445923..c2c5cc45 100644 --- a/README.md +++ b/README.md @@ -1204,6 +1204,29 @@ the load counts as at least the sum of those records, and nothing starts while they kept every CPU busy. macOS, Windows, and Linux without a systemd user manager have no scope to measure, so the load average alone decides there. +Where a systemd scope owns the command and the user manager may limit memory +(the memory controller is delegated to it), the scope also holds the command's +memory to a ceiling, with no swap, so a command that overruns stops itself +instead of the kernel's OOM killer picking a victim elsewhere on the host. A +ceiling starts at an eighth of one command's budget, 512 MiB on most hosts. +Every second the gate records what its command holds, not counting file cache: + +- Once that reaches three quarters of the ceiling, the gate raises the ceiling + by half, or by one starting ceiling if that is more or all that fits. A raise + comes out of the host's available memory, less what the other running + commands may still take of their ceilings, and less one command's budget, + which stays free for everything outside the gate. +- Once it falls below half, the ceiling drops to one starting ceiling above + what the command holds. + +What running commands may still take of their ceilings also counts as taken +when a new command asks to start, so the ceilings fit the host together. With +no room to raise a ceiling, the kernel slows the command over the last eighth +below it, and the gate says so on stderr. A command that fills the rest is +killed with everything in its scope, and the gate fails with +`... ran out of memory`. A nested gate's command stays under the enclosing +command's ceiling; a gate inside another Quest scope has a ceiling of its own. + A user's admissions on one host are recorded under `~/.cache/quest/admissions/`, whatever the caller's environment or login session. The home directory comes from the user database, not `$HOME`; diff --git a/quest/m0/gate-memory.md b/quest/m0/gate-memory.md deleted file mode 100644 index d349bced..00000000 --- a/quest/m0/gate-memory.md +++ /dev/null @@ -1,62 +0,0 @@ -# [S] A gated command cannot exhaust the host's memory - -## Goal - -On Linux hosts where the gate owns each command in a systemd scope, a -gated command that grows past its memory ceiling is throttled and then -killed as a whole group, instead of the kernel's OOM killer picking a -victim elsewhere on the host. Other gated commands, agent sessions, and -host services keep running. - -Non-goals: host-wide swap or oomd settings, which kixel-ai/nixos's -quest tree plans for the maintainer's host. - -## Plan - -Found 2026-11-12 in read-only audits of the maintainer's three Linux -hosts: the kernel OOM killer fired 5 times in 10 days on one host, killing a -47 GB `rust2-ts` inside a gate scope, a 36 GB `lean`, and a 38 GB gated -`java` while a different process was the one allocating. earlyoom killed -34 processes in a week on another, including HOL Light, lean, and an -Apalache run with a 16 GB heap. Gate scopes start with no memory limit -(`systemd-run` in `src/cli/supervise.ts`). - -Decided 2026-11-12 (maintainer, /quest-plan): - -- ✅ The gate's scopes get a memory ceiling and kill the whole group on - OOM (rejected: leaving it to host-wide oomd alone, which picks victims by - pressure, not by which command overran; rules telling agents to cap - heaps themselves). -- Milestone: ✅ m0 (rejected: m1). - -What the decision requires, found in review: - -- The ceilings of commands running at once must fit the host together: - per-command ceilings each sized from the memory available at admission - overcommit, since several commands admitted together each see the same - free memory. Partition one host budget (memory less a reserve for - sessions and services) across admitted commands, as the gate's memory - admission already counts each command's share (`COMMAND_MEMORY` in - `src/cli/gate.ts`), and admit a command only when its ceiling fits. -- A nested gate's scope is a sibling bound to its enclosing scope - (`BindsTo`), not a child cgroup, so it does not inherit the parent's - limit: its usage must come out of the enclosing command's share. - -Agent's choices, for the implementer to revisit: - -- Set `MemoryHigh` below `MemoryMax`, so a command slows under reclaim - before it is killed, and `OOMPolicy=kill`, so an OOM kill inside the - scope stops the whole command rather than leaving its wrappers running. -- Size a command's share from its measured peak once the gate has seen - it, rather than a constant (moq-dev/quest#673 dropped the matching CPU - plan); a command - that needs more fails loud with the scope's OOM event in the gate's - error, so the agent learns why. - -## Hosts - -- linux - systemd scopes carry the memory limits - -## Related - -- [Coordinators start an agent only when the host and the account can carry it](/quest/m0/agent-admission.md) - the agent-level half of host admission diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 1db92644..f911e044 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -681,8 +681,9 @@ export async function measure(host: Host, admission: { id: string; unit: string * to what the command holds, rounded down to half a `step`, so other callers count the rest of its * ceiling as promised: * - * - Once the command holds three quarters of its ceiling, its gate raises the ceiling by a quarter - * of it, or by `step` if that is more or all that fits. The raise comes from the host's available + * - Once the command holds three quarters of its ceiling, its gate raises the ceiling by half of + * it, so a command that grows fast is slowed for few samples, or by `step` if that is more or + * all that fits. The raise comes from the host's available * memory, less what the other running commands may still take of their ceilings and less * `reserve`, which stays free for everything outside the gate. Ceilings therefore never promise * more than the host has. @@ -729,7 +730,7 @@ export async function ration( .reduce((sum, entry) => sum + unused(entry), 0); // This command's own unused ceiling is already promised to it. const room = host.memory() - promised - reserve - Math.max(0, share - memory); - const wanted = Math.max(step, Math.floor(share / 4)); + const wanted = Math.max(step, Math.floor(share / 2)); next += room >= wanted ? wanted : room >= step ? step : 0; } if (mine.memory === next && mine.held === held) break; diff --git a/src/cli/main.ts b/src/cli/main.ts index 1cd95c8b..721495eb 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -134,7 +134,9 @@ export function main(argv: string[], out: Output): number | Promise { "only the raw measures do. The load limit applies only while a gated command still runs. Where a " + "systemd scope owns each running command, the load counts as at least the sum of their peak CPU " + "use in the last minute, and nothing starts while those fill every CPU, so a lull between a " + - "command's phases does not free its CPUs. Leave model calls, such as quest review, and long-running " + + "command's phases does not free its CPUs. Where the user manager may limit memory, the scope also " + + "holds the command to a memory ceiling, which the gate raises while the host has room; a command " + + "that fills its ceiling is killed whole, and the gate fails. Leave model calls, such as quest review, and long-running " + "servers out of the gate. Nested gate " + "calls run immediately. Limits come from the host; there is " + "no configuration.\n\n" + diff --git a/test/gate.test.ts b/test/gate.test.ts index 96937cd6..305281d8 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -358,17 +358,16 @@ test("a gate keeps its command's ceiling ahead of what it holds while the host h // Three quarters of the ceiling: one more step. { held: 0.8, available: 9 }, { held: 1.6, available: 9 }, + // Past 2 GiB, half of the ceiling is more than a step. { held: 2.4, available: 9 }, - { held: 3.2, available: 9 }, - // Past 4 GiB, a quarter of the ceiling is more than a step. - { held: 4, available: 9 }, + { held: 3.5, available: 9 }, // Half of it is no reason to move. - { held: 3.2, available: 9 }, + { held: 3.4, available: 9 }, // Less is: a step above what it holds. { held: 1.2, available: 9 }, { held: 0.1, available: 9 }, ]); - expect(caps).toEqual([2, 3, 4, 5, 6.25, 3, 2]); + expect(caps).toEqual([2, 3, 4.5, 6.75, 3, 2]); expect(throttled).toEqual([]); expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 2 * GiB, held: 0 }); }); From d6d1e83c641cc5d8df5a44dcdc0cc9ab9305f109 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 06:41:25 -0800 Subject: [PATCH 5/6] gate: lower a scope's memory limit to its share only once the host has no room A limit that starts small keeps Node and the JVM small, since they size their heap from it at startup. Count the limits of the slice the scopes run in as well. Co-Authored-By: Claude Fable 5.2 --- README.md | 48 +++++++----- src/cli/gate.ts | 181 ++++++++++++++++++++++++++----------------- src/cli/main.ts | 4 +- src/cli/supervise.ts | 23 ++++-- test/gate.test.ts | 102 +++++++++++++++--------- 5 files changed, 224 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index c2c5cc45..a1928584 100644 --- a/README.md +++ b/README.md @@ -1205,27 +1205,37 @@ they kept every CPU busy. macOS, Windows, and Linux without a systemd user manager have no scope to measure, so the load average alone decides there. Where a systemd scope owns the command and the user manager may limit memory -(the memory controller is delegated to it), the scope also holds the command's -memory to a ceiling, with no swap, so a command that overruns stops itself -instead of the kernel's OOM killer picking a victim elsewhere on the host. A -ceiling starts at an eighth of one command's budget, 512 MiB on most hosts. -Every second the gate records what its command holds, not counting file cache: - -- Once that reaches three quarters of the ceiling, the gate raises the ceiling - by half, or by one starting ceiling if that is more or all that fits. A raise +(the memory controller is delegated to it), each running command also has a +share of the host's memory, so a command that overruns stops itself instead of +the kernel's OOM killer picking a victim elsewhere on the host. A share starts +at an eighth of one command's budget, 512 MiB on most hosts. Every second the +gate records what its command holds, not counting file cache: + +- Once that reaches three quarters of the share, the gate raises the share by + half, or by one starting share if that is more or all that fits. A raise comes out of the host's available memory, less what the other running - commands may still take of their ceilings, and less one command's budget, + commands may still take of their shares, and less one command's budget, which stays free for everything outside the gate. -- Once it falls below half, the ceiling drops to one starting ceiling above - what the command holds. - -What running commands may still take of their ceilings also counts as taken -when a new command asks to start, so the ceilings fit the host together. With -no room to raise a ceiling, the kernel slows the command over the last eighth -below it, and the gate says so on stderr. A command that fills the rest is -killed with everything in its scope, and the gate fails with -`... ran out of memory`. A nested gate's command stays under the enclosing -command's ceiling; a gate inside another Quest scope has a ceiling of its own. +- Once it falls below half, the share drops to one starting share above what + the command holds. + +What running commands may still take of their shares also counts as taken when +a new command asks to start, so the shares fit the host together. Available +memory here is also bounded by any memory limit on `background.slice` or the +cgroups that hold it. + +While raises fit, the scope's own limit stays where it started: the host's +memory less one command's budget, with no swap. A runtime that sizes its heap +from its cgroup's limit when it starts, as Node and the JVM do, would stay +small under a limit that only grows later. A command can therefore pass its +share within a second, into the budget kept free, and its share then follows +what it took. Once no raise fits, the gate lowers the scope's limit to the +share and says so on stderr: the kernel slows the command over the last eighth +below it and, if the command fills the rest, kills everything in its scope, +and the gate fails with `... ran out of memory`. The limit lifts again once +the command wants no more or a raise fits. A nested gate's command stays under +the enclosing command's limit; a gate inside another Quest scope has a share +of its own. A user's admissions on one host are recorded under `~/.cache/quest/admissions/`, whatever the caller's environment or diff --git a/src/cli/gate.ts b/src/cli/gate.ts index f911e044..9b96accc 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -20,7 +20,16 @@ import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; import { stubbed } from "./job"; -import { type Ceiling, capped, limit, questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; +import { + type Ceiling, + capped, + limit, + managerGroup, + questScope, + scopeGroup, + startTime, + systemdScopes, +} from "./supervise"; /** * One admitted command: a unique id, the wall-clock time it was admitted, and the process ID of @@ -29,8 +38,8 @@ import { type Ceiling, capped, limit, questScope, scopeGroup, startTime, systemd * waiting for its turn instead, as of `at`: it holds a place in the queue, the ledger's order of * queued entries, and no slot. `cores` is the most CPUs the command kept busy over any * SETTLE_MS of the last RECENT_MS, where its gate measures it (see `measure`). `memory` is the - * ceiling its scope holds its memory to, in bytes, where the gate caps it, and `held` what its - * command held of that when last sampled (see `ration`). + * share of the host's memory its command counts as taking, in bytes, where the gate can hold it + * to one, and `held` what its command held of that when last sampled (see `ration`). */ export interface Admission { id: string; @@ -87,28 +96,27 @@ export interface Limits { /** The least available memory, in bytes, that still admits a command. */ memory: number; /** - * The memory ceiling an admitted command's scope starts with, in bytes, and the least its gate - * raises it by (see `ration`); undefined where no scope caps the command. + * The share of memory an admitted command starts with, in bytes, and the least its gate raises + * it by (see `ration`); undefined where no scope can hold the command to one. */ ceiling?: number; } -/** A ceiling starts at this fraction of one command's memory, so what a small command never uses stays free. */ +/** A share starts at this fraction of one command's memory, so what a small command never uses stays free. */ const CEILING_PARTS = 8; -/** How often a gate compares its command's memory with its ceiling. */ +/** How often a gate compares its command's memory with its share. */ export const CEILING_MS = 1000; /** - * The scope limits for a ceiling of `share` bytes: the kernel slows the command over the last - * eighth below it, ever harder the nearer it gets, so its gate has time to raise both while the - * host has room, and kills it only once it fills the rest. + * The scope limits that hold a command to `share` bytes: the kernel slows the command over the + * last eighth below it, ever harder the nearer it gets, and kills it once it fills the rest. */ export function bounds(share: number): Ceiling { return { high: share - Math.floor(share / 8), max: share }; } -/** What a running command may still take of its ceiling: memory promised to it, so not free for another. */ +/** What a running command may still take of its share: memory promised to it, so not free for another. */ function unused(admission: Admission): number { return Math.max(0, (admission.memory ?? 0) - (admission.held ?? 0)); } @@ -152,9 +160,23 @@ export interface Precedence { /** The memory one gated build or test stack is assumed to need. */ export const COMMAND_MEMORY = 4 * 2 ** 30; +let slice: string[] | undefined; + +/** + * The slice the gate's scopes run in (see `background`), where a user manager has one: a limit on + * it or above it bounds every gated command, though the gate itself runs outside it. + */ +function scopes(): string[] { + if (slice === undefined) { + const manager = managerGroup(); + slice = manager === undefined ? [] : [`${manager}/background.slice`]; + } + return slice; +} + /** Usable memory: physical RAM, or a smaller cgroup/container limit. */ function usableMemory(): number { - if (process.platform === "linux") return linuxMemory().total; + if (process.platform === "linux") return linuxMemory(undefined, undefined, scopes()).total; const constrained = process.constrainedMemory?.() ?? 0; return constrained > 0 ? Math.min(totalmem(), constrained) : totalmem(); } @@ -189,8 +211,15 @@ function cgroupStat(group: string, key: string): number { /** A gate scope's cgroup directory name. */ const GATE_SCOPE = /^quest-gate-[0-9a-f-]+\.scope$/; -/** Count reclaimable host cache and the headroom of every enclosing memory cgroup, except a gate's own ceiling. */ -export function linuxMemory(proc = "/proc", cgroups = "/sys/fs/cgroup"): { total: number; available: number } { +/** + * Count reclaimable host cache and the headroom of every enclosing memory cgroup, except a gate's + * own ceiling, and of the unified cgroups `beside` and those enclosing them. + */ +export function linuxMemory( + proc = "/proc", + cgroups = "/sys/fs/cgroup", + beside: string[] = [], +): { total: number; available: number } { const meminfo = readFileSync(join(proc, "meminfo"), "utf8"); const bytes = (key: string) => { const value = meminfo.match(new RegExp(`^${key}:\\s+(\\d+) kB$`, "m"))?.[1]; @@ -199,7 +228,9 @@ export function linuxMemory(proc = "/proc", cgroups = "/sys/fs/cgroup"): { total }; let total = bytes("MemTotal"); let available = bytes("MemAvailable"); - for (const line of readFileSync(join(proc, "self/cgroup"), "utf8").trim().split("\n")) { + // A command that runs in another unified cgroup, as in a scope of its own, is bounded there too. + const elsewhere = beside.map((path) => `0::${path}`); + for (const line of [...readFileSync(join(proc, "self/cgroup"), "utf8").trim().split("\n"), ...elsewhere]) { const [, controllers, path] = line.match(/^\d+:([^:]*):(.*)$/) ?? []; if (controllers === undefined || path === undefined) throw new Error("gate cannot read Linux memory cgroups"); const unified = controllers === ""; @@ -402,7 +433,7 @@ export function fileHost(directory: string, now = Date.now): Host { process.platform === "darwin" ? Infinity : process.platform === "linux" - ? linuxMemory().available + ? linuxMemory(undefined, undefined, scopes()).available : process.availableMemory(), now, running: (admission) => @@ -587,7 +618,7 @@ export async function admit( : running.size > 0 ? Math.max(host.load(), used) : Math.min(host.load(), limits.load); - // Running commands' ceilings must fit the host together, so what each may still take of its + // Running commands' shares must fit the host together, so what each may still take of its // own is not free, however long it has run. const memory = host.memory() - [...running].reduce((sum, admission) => sum + unused(admission), 0); let wait = SETTLE_MS; @@ -598,7 +629,7 @@ export async function admit( // the host has room for them all. const taken = recent.length + ahead; // A recent command has yet to show its memory, so it counts as one command's, less what - // its ceiling already took above. + // its share already took above. const lagging = recent.reduce( (sum, admission) => sum + Math.max(0, limits.memory - (running.has(admission) ? unused(admission) : 0)), ahead * limits.memory, @@ -676,33 +707,38 @@ export async function measure(host: Host, admission: { id: string; unit: string } /** - * Until `signal` aborts, keep the memory ceiling of the admitted command in systemd scope `unit` - * ahead of what it holds, while the host has room. Every CEILING_MS, the admission's `held` is set - * to what the command holds, rounded down to half a `step`, so other callers count the rest of its - * ceiling as promised: + * Until `signal` aborts, account for the memory of the admitted command in systemd scope `unit`, + * and hold it to its share once the host has no room for more. Every CEILING_MS, the admission's + * `held` is set to what the command holds, rounded down to half a `step`, and its `memory` to its + * share, so other callers count the rest of the share as promised: * - * - Once the command holds three quarters of its ceiling, its gate raises the ceiling by half of - * it, so a command that grows fast is slowed for few samples, or by `step` if that is more or - * all that fits. The raise comes from the host's available - * memory, less what the other running commands may still take of their ceilings and less - * `reserve`, which stays free for everything outside the gate. Ceilings therefore never promise + * - Once the command holds three quarters of its share, its gate raises the share by half of it, + * or by `step` if that is more or all that fits. The raise comes from the host's available + * memory, less what the other running commands may still take of their shares and less + * `reserve`, which stays free for everything outside the gate. Shares therefore never promise * more than the host has. - * - Once it holds less than half, the ceiling falls to a `step` above what it holds, so a peak the + * - Once it holds less than half, the share falls to a `step` above what it holds, so a peak the * command has left is free for others. * - * With no room, the ceiling stays: the kernel slows the command there and kills it once it fills - * the rest (see `bounds`), so the command that overran stops, not another. `throttled` is called - * with the ceiling each time the command starts being held at one. + * While raises fit, the scope keeps the limit it started with, `most`: a runtime that sizes its + * heap from its cgroup's limit when it starts, as Node and the JVM do, would stay small under a + * limit that only grows later. A command can so pass its share between samples, into `reserve`; + * its share then follows what it took. With no room for a raise, the scope's limits drop to the + * share (see `bounds`): the kernel slows the command there and kills it once it fills the rest, + * so the command that overran stops, not another. `throttled` is called with the share each time + * the command starts being held to one; the limits return to `most` once it no longer wants more + * or a raise fits. */ export async function ration( host: Host, - admission: { id: string; unit: string; step: number; reserve: number }, + admission: { id: string; unit: string; step: number; reserve: number; most: number }, signal: AbortSignal, - throttled?: (ceiling: number) => void, + throttled?: (share: number) => void, ) { - const { id, unit, step, reserve } = admission; + const { id, unit, step, reserve, most } = admission; let share = step; - let slowed = false; + // The share the scope's limits hold the command to, while they do. + let clamped: number | undefined; for (;;) { try { await host.sleep(CEILING_MS, signal); @@ -713,23 +749,22 @@ export async function ration( const memory = host.usage(unit)?.memory; if (memory === undefined) continue; const held = memory - (memory % Math.ceil(step / 2)); - const fitted = Math.max(step, Math.ceil(memory / step) * step + step); - // Lowered before it is recorded, and raised after, so the ledger never promises less than the scope allows. - if (memory < share / 2 && fitted < share) { - share = fitted; - host.cap(unit, bounds(share)); - } + const fitted = Math.ceil(memory / step) * step; + // What it already took is its share, whether or not it fit. + if (memory > share) share = fitted; + else if (memory < share / 2) share = Math.min(share, fitted + step); + const wants = memory > share - share / 4; let next = share; for (let ledger = host.ledger(); ; ledger = host.ledger()) { const mine = ledger.admissions.find((entry) => entry.id === id); if (mine === undefined) throw new Error(`gate lost its admission ${id} from the ledger`); next = share; - if (memory > share - share / 4) { + if (wants) { const promised = ledger.admissions .filter((entry) => entry !== mine && !entry.queued && host.running(entry)) .reduce((sum, entry) => sum + unused(entry), 0); - // This command's own unused ceiling is already promised to it. - const room = host.memory() - promised - reserve - Math.max(0, share - memory); + // This command's own unused share is already promised to it. + const room = Math.min(most - share, host.memory() - promised - reserve - (share - memory)); const wanted = Math.max(step, Math.floor(share / 2)); next += room >= wanted ? wanted : room >= step ? step : 0; } @@ -743,14 +778,12 @@ export async function ration( ) break; } - if (next > share) { - share = next; - host.cap(unit, bounds(share)); - slowed = false; - } else if (memory >= bounds(share).high) { - if (!slowed) throttled?.(share); - slowed = true; - } else slowed = false; + const limit = wants && next === share ? share : undefined; + share = next; + if (limit === clamped) continue; + host.cap(unit, limit === undefined ? { high: most, max: most } : bounds(limit)); + if (limit !== undefined) throttled?.(limit); + clamped = limit; } } @@ -929,28 +962,30 @@ const TOKEN = 3; * call inside a Quest scope runs its command in a background scope bound to it. Linux owns descendants through a systemd scope, and settles for a process group, * like macOS, without a systemd user manager. Windows owns them through a Job Object. A systemd * scope also lets the gate measure its command's CPU use (see `measure`), so `unit` names it. - * Where the user manager can limit memory, the scope starts with a ceiling of `ceiling` bytes, a - * fraction of `memory`, one command's, which the gate then raises (see `ration`). A nested call's - * command stays in the admitted command's cgroup, under its ceiling; every scope the gate starts - * has an admission, and so a ceiling, of its own. + * Where the user manager can limit memory, the scope's memory is limited to `most` bytes, all but + * one command's `memory` of the host's `total`, and `ceiling`, a fraction of `memory`, is the share + * its gate starts the command with (see `ration`). A nested call's command stays in the admitted + * command's cgroup, under its limits; every scope the gate starts has an admission, and so a share, + * of its own. */ export function owner( env = process.env, cgroup = "/proc/self/cgroup", - memory = hostLimits().memory, -): { run: Owner; unit?: string; ceiling?: number } { + host = { memory: hostLimits().memory, total: usableMemory() }, +): { run: Owner; unit?: string; ceiling?: number; most?: number } { // A Quest scope around this process holds the command too, since it shares the cgroup or an ancestor. const enclosing = process.platform === "linux" ? questScope(cgroup) : undefined; if (env[ADMITTED] === "1") return { run: env[GROUP] === "1" || enclosing ? inside : direct }; if (process.platform === "win32") return { run: job }; const unit = `quest-gate-${randomUUID()}.scope`; const held = () => { - const ceiling = Math.floor(memory / CEILING_PARTS); - return capped() && ceiling > 0 ? { ceiling, limits: bounds(ceiling) } : {}; + const ceiling = Math.floor(host.memory / CEILING_PARTS); + const most = Math.floor(host.total - host.memory); + return capped() && ceiling > 0 && most >= ceiling ? { ceiling, most, limits: { high: most, max: most } } : {}; }; if (enclosing) { - const { ceiling, limits } = held(); - return { run: bound(unit, enclosing.unit, limits), unit, ceiling }; + const { limits, ...shares } = held(); + return { run: bound(unit, enclosing.unit, limits), unit, ...shares }; } if (process.platform !== "linux") return { run: group }; try { @@ -958,8 +993,8 @@ export function owner( } catch { return { run: group }; } - const { ceiling, limits } = held(); - return { run: scope(unit, limits), unit, ceiling }; + const { limits, ...shares } = held(); + return { run: scope(unit, limits), unit, ...shares }; } const notices: Record = { @@ -977,7 +1012,10 @@ export async function gate(command: string[], options: { quick?: boolean } = {}) const cancel = interrupts(); try { const limits = hostLimits(); - const { run, unit, ceiling } = owner(process.env, undefined, limits.memory); + const { run, unit, ceiling, most } = owner(process.env, undefined, { + memory: limits.memory, + total: usableMemory(), + }); let admitted: { host: Host; id: string } | undefined; // A nested invocation executes inside the already admitted command. if (process.env[ADMITTED] !== "1") { @@ -1004,16 +1042,21 @@ export async function gate(command: string[], options: { quick?: boolean } = {}) const { host, id } = admitted ?? {}; const throttled = (share: number) => process.stderr.write( - `quest gate: the command is held at its memory ceiling of ${(share / 2 ** 30).toFixed(2)} GiB ` + - "until the host has room to raise it\n", + `quest gate: the host has no memory to spare, so the command is held to ${(share / 2 ** 30).toFixed(2)} GiB ` + + "until it does\n", ); const measuring = host !== undefined && id !== undefined && unit !== undefined ? Promise.all([ measure(host, { id, unit }, stop.signal), - ceiling === undefined + ceiling === undefined || most === undefined ? undefined - : ration(host, { id, unit, step: ceiling, reserve: limits.memory }, stop.signal, throttled), + : ration( + host, + { id, unit, step: ceiling, reserve: limits.memory, most }, + stop.signal, + throttled, + ), ]).then( () => undefined, (error: unknown) => ({ error }), diff --git a/src/cli/main.ts b/src/cli/main.ts index 721495eb..950eebbf 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -135,8 +135,8 @@ export function main(argv: string[], out: Output): number | Promise { "systemd scope owns each running command, the load counts as at least the sum of their peak CPU " + "use in the last minute, and nothing starts while those fill every CPU, so a lull between a " + "command's phases does not free its CPUs. Where the user manager may limit memory, the scope also " + - "holds the command to a memory ceiling, which the gate raises while the host has room; a command " + - "that fills its ceiling is killed whole, and the gate fails. Leave model calls, such as quest review, and long-running " + + "gives the command a share of memory, which the gate raises while the host has room; once it has " + + "none, the command is held to its share, and killed whole if it fills it, failing the gate. Leave model calls, such as quest review, and long-running " + "servers out of the gate. Nested gate " + "calls run immediately. Limits come from the host; there is " + "no configuration.\n\n" + diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index c1f5eebc..b13a2d04 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -272,11 +272,9 @@ async function stopBound(unit: string): Promise { /** Whether the user manager may limit a scope's memory: the memory controller is delegated to it. */ export function capped(): boolean { + const manager = managerGroup(); + if (manager === undefined) return false; try { - const manager = execFileSync("systemctl", ["--user", "show", "--property=ControlGroup", "--value"], { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }).trim(); return readFileSync(join("/sys/fs/cgroup", manager, "cgroup.controllers"), "utf8") .trim() .split(" ") @@ -286,6 +284,19 @@ export function capped(): boolean { } } +/** The user manager's cgroup, as a path below the cgroup mount, which holds its slices; undefined without one. */ +export function managerGroup(): string | undefined { + try { + const group = execFileSync("systemctl", ["--user", "show", "--property=ControlGroup", "--value"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + return group || undefined; + } catch { + return undefined; + } +} + /** Move a running scope's memory ceiling. A scope that already ended has none to move. */ export function limit(unit: string, memory: Ceiling): void { try { @@ -523,8 +534,8 @@ function contain( if (failure) throw failure; if (code !== 0 && !interrupted && describe.exhausted()) throw new Exhausted( - `${describe.name} ran out of memory: the kernel killed its command at its memory ceiling, ` + - "which its gate raises only while the host has free memory", + `${describe.name} ran out of memory: the kernel killed its command at its memory limit, ` + + "which its gate lowers to what the command holds once the host has none to spare", ); if (!admitted && !interrupted) throw new Error( diff --git a/test/gate.test.ts b/test/gate.test.ts index 305281d8..0cdb4f9f 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -314,10 +314,10 @@ test("what running commands may still take of their ceilings is not free, howeve }); /** - * Run `ration` for a command with a 1 GiB ceiling that moves by as much, on a host that keeps - * 1 GiB free, over `samples`: what the command holds and what the host has available at each - * CEILING_MS. Returns each ceiling the scope was moved to, each one it was held at, and its - * admission. + * Run `ration` for a command whose share starts at 1 GiB and moves by as much, on a host that + * keeps 1 GiB free and gives one command at most 64, over `samples`: what the command holds and + * what the host has available at each CEILING_MS. Returns its share after each sample, each limit + * its scope was moved to, each share it was held to, and its admission. */ async function rationed(samples: { held: number; available: number }[], others: Admission[] = []) { const state = { now: 0, load: 0, memory: Infinity }; @@ -327,13 +327,20 @@ async function rationed(samples: { held: number; available: number }[], others: host.usage = () => ({ cpu: 0, memory: held }); const caps: number[] = []; host.cap = (_unit, { high, max }) => { - expect(high).toBe(max - max / 8); - // The ledger already promises a raised ceiling, and still promises a lowered one. - expect(host.ledger().admissions[0].memory).toBeGreaterThanOrEqual(max); + // Back at the most one command may hold, or held to its share and slowed over its last eighth. + if (high === max) expect(max).toBe(64 * GiB); + else { + expect(high).toBe(max - max / 8); + expect(host.ledger().admissions[0].memory).toBe(max); + } caps.push(max / GiB); }; const stop = new AbortController(); + const shares: number[] = []; + let sampled = false; host.sleep = async () => { + if (sampled) shares.push(host.ledger().admissions[0].memory! / GiB); + sampled = true; const sample = samples.shift(); if (sample === undefined) { stop.abort(); @@ -345,46 +352,54 @@ async function rationed(samples: { held: number; available: number }[], others: const throttled: number[] = []; await ration( host, - { id: "build", unit: "quest-gate-build.scope", step: GiB, reserve: GiB }, + { id: "build", unit: "quest-gate-build.scope", step: GiB, reserve: GiB, most: 64 * GiB }, stop.signal, - (ceiling) => throttled.push(ceiling / GiB), + (share) => throttled.push(share / GiB), ); - return { caps, throttled, mine: host.ledger().admissions[0] }; + return { shares, caps, throttled, mine: host.ledger().admissions[0] }; } -test("a gate keeps its command's ceiling ahead of what it holds while the host has room", async () => { - const { caps, throttled, mine } = await rationed([ - { held: 0.25, available: 9 }, - // Three quarters of the ceiling: one more step. - { held: 0.8, available: 9 }, - { held: 1.6, available: 9 }, - // Past 2 GiB, half of the ceiling is more than a step. - { held: 2.4, available: 9 }, - { held: 3.5, available: 9 }, +test("a gate keeps its command's share ahead of what it holds while the host has room", async () => { + const { shares, caps, throttled, mine } = await rationed([ + { held: 0.25, available: 99 }, + // Three quarters of the share: one more step. + { held: 0.8, available: 99 }, + { held: 1.6, available: 99 }, + // Past 2 GiB, half of the share is more than a step. + { held: 2.4, available: 99 }, + { held: 3.5, available: 99 }, // Half of it is no reason to move. - { held: 3.4, available: 9 }, + { held: 3.4, available: 99 }, // Less is: a step above what it holds. - { held: 1.2, available: 9 }, - { held: 0.1, available: 9 }, + { held: 1.2, available: 99 }, + { held: 0.1, available: 99 }, + // What it took between samples is its share, and it wants more. + { held: 7.5, available: 99 }, + // One command never gets more than the host gives one. + { held: 63.5, available: 99 }, ]); - expect(caps).toEqual([2, 3, 4.5, 6.75, 3, 2]); - expect(throttled).toEqual([]); - expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 2 * GiB, held: 0 }); + expect(shares).toEqual([1, 2, 3, 4.5, 6.75, 6.75, 3, 2, 12, 64]); + // The scope keeps the limit it started with, so a runtime never sizes itself to a small one. + expect(caps).toEqual([64]); + expect(throttled).toEqual([64]); + expect(mine).toEqual({ id: "build", at: 0, ...me, memory: 64 * GiB, held: 63.5 * GiB }); }); -test("with no room, a ceiling stays until other commands free memory", async () => { - // Another command may still take 3 GiB of its ceiling, which is not this one's to grow into. +test("with no room, a command is held to its share until other commands free memory", async () => { + // Another command may still take 3 GiB of its share, which is not this one's to grow into. const other = { id: "other", at: 0, ...me, memory: 4 * GiB, held: GiB }; - const { caps, throttled, mine } = await rationed( + const { shares, caps, throttled, mine } = await rationed( [ // 4.7 GiB less the 3 GiB promised, the 1 GiB kept free, and its own 0.2 GiB left: no step. { held: 0.8, available: 4.7 }, - // Held where the kernel slows it: reported once, not at every sample. + // Still held: reported once, not at every sample. { held: 0.9, available: 4.6 }, - { held: 0.95, available: 4.6 }, // A step now fits, though a queued caller's place and a finished command hold nothing. { held: 0.95, available: 5.1 }, - { held: 1.9, available: 4.2 }, + // It took more than its share before the next sample, and nothing more fits. + { held: 2.6, available: 4.2 }, + // It no longer wants more, so its scope's limits lift. + { held: 2.1, available: 4.2 }, ], [ other, @@ -392,12 +407,13 @@ test("with no room, a ceiling stays until other commands free memory", async () { id: "gone", at: 0, pid: 2 ** 22 + 1, memory: 64 * GiB }, ], ); - expect(caps).toEqual([2]); - expect(throttled).toEqual([1, 2]); - expect(mine).toMatchObject({ memory: 2 * GiB, held: 1.5 * GiB }); + expect(shares).toEqual([1, 1, 2, 3, 3]); + expect(caps).toEqual([1, 64, 3, 64]); + expect(throttled).toEqual([1, 3]); + expect(mine).toMatchObject({ memory: 3 * GiB, held: 2 * GiB }); }); -test("a ceiling slows its command over the last eighth before it is killed", () => { +test("a held command is slowed over the last eighth of its share before it is killed", () => { expect(bounds(GiB)).toEqual({ high: GiB - GiB / 8, max: GiB }); }); @@ -803,6 +819,14 @@ test("Linux memory reads the container cgroup mounted as the root", () => { writeFileSync(join(cgroups, gated, "memory.current"), String(GiB)); writeFileSync(join(proc, "self/cgroup"), `0::/${gated}\n`); expect(linuxMemory(proc, cgroups)).toEqual(limited); + // A limit on the slice a command's own scope runs in bounds it, though the caller is outside it. + writeFileSync(join(cgroups, "background.slice/memory.max"), String(4 * GiB)); + writeFileSync(join(cgroups, "background.slice/memory.current"), String(3 * GiB)); + writeFileSync(join(cgroups, "background.slice/memory.stat"), `inactive_file ${GiB}\n`); + writeFileSync(join(proc, "self/cgroup"), "0::/session.scope\n"); + expect(linuxMemory(proc, cgroups, ["/background.slice"])).toEqual({ total: 4 * GiB, available: 2 * GiB }); + // A slice no unit has started in yet has no cgroup; what holds it still bounds it. + expect(linuxMemory(proc, cgroups, ["/absent.slice"])).toEqual(limited); }); test("every caller on this host shares one ledger, whatever its environment or sessions", () => { @@ -1245,7 +1269,7 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { const wait = `until [ -e ${quote(raised)} ]; do sleep 0.05; done`; const limits = `cd /sys/fs/cgroup$(cut -d: -f3 /proc/self/cgroup) && cat memory.high memory.max memory.swap.max memory.oom.group`; const result = scope(unit, { high: 192 * MiB, max: 256 * MiB })( - ["/bin/sh", "-c", `${limits} > ${quote(report)}; ${wait}; ${limits} >> ${quote(report)}`], + ["/bin/sh", "-c", `${limits} > ${quote(report)}; ${wait}; ${limits} >> ${quote(report)}; exit 7`], process.env, cancellation().cancel, ); @@ -1253,7 +1277,7 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { expect(host.usage(unit)?.memory).toBeLessThan(192 * MiB); host.cap(unit, { high: 448 * MiB, max: 512 * MiB }); writeFileSync(raised, ""); - expect(await result).toBe(0); + expect(await result).toBe(7); expect(readFileSync(report, "utf8").trim().split("\n").map(Number)).toEqual([ ...[192 * MiB, 256 * MiB, 0, 1], ...[448 * MiB, 512 * MiB, 0, 1], @@ -1289,7 +1313,9 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { process.env, cancellation().cancel, ); - await expect(run).rejects.toThrow(/ran out of memory: the kernel killed its command/); + await expect(run).rejects.toThrow( + /ran out of memory: the kernel killed its command at its memory limit/, + ); await expect(run).rejects.toBeInstanceOf(Exhausted); // The wrapper and its other child died with the hog, and the scope beside it still runs. await until(() => stopped(Number(readFileSync(pid, "utf8")))); From 9a0b59c48d51297220f7937938f7bbc4a14edbaf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 06:44:37 -0800 Subject: [PATCH 6/6] gate: sample a command's memory four times a second Co-Authored-By: Claude Fable 5.2 --- README.md | 9 +++++---- src/cli/gate.ts | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a1928584..1d98be0e 100644 --- a/README.md +++ b/README.md @@ -1208,8 +1208,8 @@ Where a systemd scope owns the command and the user manager may limit memory (the memory controller is delegated to it), each running command also has a share of the host's memory, so a command that overruns stops itself instead of the kernel's OOM killer picking a victim elsewhere on the host. A share starts -at an eighth of one command's budget, 512 MiB on most hosts. Every second the -gate records what its command holds, not counting file cache: +at an eighth of one command's budget, 512 MiB on most hosts. Four times a +second the gate records what its command holds, not counting file cache: - Once that reaches three quarters of the share, the gate raises the share by half, or by one starting share if that is more or all that fits. A raise @@ -1228,8 +1228,9 @@ While raises fit, the scope's own limit stays where it started: the host's memory less one command's budget, with no swap. A runtime that sizes its heap from its cgroup's limit when it starts, as Node and the JVM do, would stay small under a limit that only grows later. A command can therefore pass its -share within a second, into the budget kept free, and its share then follows -what it took. Once no raise fits, the gate lowers the scope's limit to the +share between two samples, into the budget kept free, and its share then +follows what it took. Commands that together take more than that budget within +a quarter of a second can still exhaust the host. Once no raise fits, the gate lowers the scope's limit to the share and says so on stderr: the kernel slows the command over the last eighth below it and, if the command fills the rest, kills everything in its scope, and the gate fails with `... ran out of memory`. The limit lifts again once diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 9b96accc..aa5dae0b 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -105,8 +105,12 @@ export interface Limits { /** A share starts at this fraction of one command's memory, so what a small command never uses stays free. */ const CEILING_PARTS = 8; -/** How often a gate compares its command's memory with its share. */ -export const CEILING_MS = 1000; +/** + * How often a gate compares its command's memory with its share. Until a sample finds the host out + * of room, nothing but the memory kept free absorbs what commands take past their shares, so the + * interval bounds how much that can be. + */ +export const CEILING_MS = 250; /** * The scope limits that hold a command to `share` bytes: the kernel slows the command over the