From 2a176227a1097950210ae61f9425c18dd0ea4c15 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 09:50:12 -0800 Subject: [PATCH 01/13] chore(quest): claim quest/m0/benchmark/gating.md Quest-Claim: 211bbd6224d4fa45 From 7da33b33d50bc5ada6573561be9e9f772575e557 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:14:47 -0800 Subject: [PATCH 02/13] feat(records): record what each session gated and the CPU its shell calls used Each session record gains `gated`, its outermost `quest gate` runs with command, queue, run, and the CPU milliseconds the gate read from its systemd scope, and `shell`, its shell calls grouped where they overlapped with the CPU the session's own cgroup used meanwhile. The gate logs one line per run, from which the record also takes its gate-queue and gate-run waits. A measured scope keeps a holder process until the gate reads its cpu.stat, since systemd otherwise removed the cgroup first about one run in twenty. Quest's PreToolUse, new PostToolUse, and PostToolUseFailure hooks, and Claude Code's Stop, sample the session cgroup's counter into $XDG_STATE_HOME/quest/calls, only where that cgroup holds the session alone. The mistake audit reads both lists. Co-Authored-By: Claude Opus 5.5 --- README.md | 37 +++++- quest/m0/benchmark/gating.md | 85 ------------- quest/m0/gate-quick.md | 5 - quest/m0/mistake-audit.md | 17 ++- src/cli/calls.ts | 227 +++++++++++++++++++++++++++++++++++ src/cli/gate.ts | 58 ++++----- src/cli/hooks.ts | 86 +++++++++++-- src/cli/main.ts | 10 +- src/cli/records.ts | 35 +++++- src/cli/send.ts | 4 + src/cli/supervise.ts | 126 +++++++++++++------ src/cli/usage.ts | 10 +- src/cli/waits.ts | 112 ++++++++++------- src/core/index.ts | 5 + src/core/record.ts | 101 ++++++++++++++++ test/calls.test.ts | 173 ++++++++++++++++++++++++++ test/gate.test.ts | 44 ++++++- test/hooks.test.ts | 83 ++++++++++++- test/records.test.ts | 198 +++++++++++++++++++++++++++++- test/send.test.ts | 1 + test/split.test.ts | 2 + test/usage.test.ts | 1 + 22 files changed, 1187 insertions(+), 233 deletions(-) delete mode 100644 quest/m0/benchmark/gating.md create mode 100644 src/cli/calls.ts create mode 100644 test/calls.test.ts diff --git a/README.md b/README.md index d940db86..18c720ea 100644 --- a/README.md +++ b/README.md @@ -1724,7 +1724,7 @@ latest upload, since one sender sends a session's uploads in order. The type is `parseUpload`, which refuse a malformed record whole: ```json -{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null}],"split":null} +{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null},{"kind":"gate-run","start":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","until":null}],"gated":[{"command":"just test","start":"2026-11-14T00:22:07.386Z","admitted":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","cpu":1214300}],"shell":[{"calls":[{"command":"git status --short","start":"2026-11-14T00:21:58.004Z","end":"2026-11-14T00:21:58.051Z"}],"cpu":30}],"split":null} ``` - `pulls` are the PRs it worked on, attributed as `quest usage` attributes @@ -1756,6 +1756,32 @@ latest upload, since one sender sends a session's uploads in order. The type is session on another host in `quest usage --ssh`, or one whose transcript has lines token-max could not read either. Like token costs, waits count only those that ended since the window's start. +- `gated` lists each outermost `quest gate` run: its `command`, cut to its + first line and 200 characters, when it asked to start, when it was + `admitted` (null when it ended queued), when it ended, and the CPU + milliseconds its command used, read from its systemd scope once it exits, + or null where the gate owns no scope, as on macOS or Windows. The gate logs + each run as one line among the waits, and the record takes its + `gate-queue` and `gate-run` waits from it. `nix build` counts little here, + since `nix-daemon` builds outside the scope. +- `shell` lists its shell calls, each cut the same way, in groups: calls that + overlapped, in parallel or in the background, are one group. A group's + `cpu` is the milliseconds the session's own cgroup used from its first + call's start to its last call's end, which counts what ran outside the + gate, and the harness meanwhile, but no gated command, whose scope is + elsewhere. Quest's tool hooks sample that counter around each call into + `$XDG_STATE_HOME/quest/calls`, kept like the waits, and only where the + cgroup holds the session alone: one tree of processes whose top is the + harness, as when an app such as T3 Code starts each agent session in a + systemd scope of its own. A terminal's scope holds the terminal's shell + above the harness, so its groups, like those on macOS, have a null `cpu`. + A call Claude Code moved to the background runs until the first turn end + that no longer lists it; a call that never returned, as one denied, is + left out. Codex's calls end when the tool returns, so a command it left + running counts only until then. A group shared by sessions in one cgroup, + as a sub-agent's with its parent's, is in each of their records. Both + lists are null for a session on another host, and count what ended since + the window's start. - `split` is where the rest of its time went, as the next section describes; null until every step is labeled, and for a session on another host. @@ -2243,10 +2269,13 @@ the ID Claude Code gave it on backgrounding it, so a sibling's gate does not hold an agent. Claude Code lists a command only while it runs, so the refusal ends with the gate. Codex runs no such hook. Once the turn can end, Claude Code's and Codex's `Stop` hooks queue the session's -[record](#session-records). +[record](#session-records). For that record, the `PreToolUse` hook and a +`PostToolUse` hook on shell commands, with Claude Code's `PostToolUseFailure`, +also sample the CPU each shell call used, and Claude Code's `Stop` and +`SubagentStop` which background commands still run. -`quest` must be on the `PATH` the agent starts with, as for the worktree `--body`, a `body=` field, -`git commit -m`, `--body-file -` with a here-document, or a body file. A file +It checks a body given by `--body`, a `body=` field, `git commit -m`, +`--body-file -` with a here-document, or a body file. A file is read only when its command is the only one on the command line; after another command, or piped into `gh`, a body passes unread, since it may hold something else by the time the command runs. `quest context` still prints these rules, diff --git a/quest/m0/benchmark/gating.md b/quest/m0/benchmark/gating.md deleted file mode 100644 index 852ce8c4..00000000 --- a/quest/m0/benchmark/gating.md +++ /dev/null @@ -1,85 +0,0 @@ -# [L] Records show what each session gated, and what it should have - -## Goal - -Each session's record lists the commands it ran through `quest gate`, each -with its wait, CPU, and wall time, and the CPU each of its other shell -calls used. An audit can then see both kinds of gating mistake: a gated -command that barely used the CPU, such as `quest gate quest review`, and a -CPU-heavy command run outside the gate. `quest usage --json` prints them, -the records Kixel Code stores carry them, and the -[mistake audit](/quest/m0/mistake-audit.md) reads them in each weekly run, -its Plan updated to say so. - -Non-goals: refusing any command at the gate or in a hook; changing how the -gate admits; per-process host monitoring, which Kixel Code's host report -owns. - -## Plan - -Decided 2026-11-15 (maintainer, `/quest-plan`), from an audit of 14 days -of Claude and Codex transcripts on the maintainer's three Linux hosts, -joined to the gate's wait log and each gate scope's CPU accounting in the -systemd journal: - -- Gate by CPU, not run time? ✅ Yes: `quest context`, `quest gate --help`, - and the README now say commands that may use significant CPU go through - the gate, and `quest review`, servers, and formatters stay out - (rejected: a two-sentence wording explaining each exception). The chosen - sentence lost "browsers" and "which queues until the host has room" to - fit the 10,000 characters a hook's context may hold. -- How are gating mistakes caught? ✅ Measured here and audited weekly - (maintainer: "Instead of making quest gate impossible with quest review, - add this to the metrics we're planning"). Rejected: a `quest hook` rule - denying `quest gate quest review` before it queues. -- How is an ungated command's CPU found? ✅ Quest's `PreToolUse` and - `PostToolUse` hooks read the session's cgroup CPU counter around each - shell call and record the difference. Rejected: matching ungated - commands by name against gated runs of the same command, which misses - commands never gated and cannot tell a heavy run from a light one; both - together, until the audit shows the fallback is needed. - -Agent's choices, for the implementer to revisit: - -- Gated side: the gate already logs each outermost run's queue and run - times per session under `$XDG_STATE_HOME/quest/waits`. Add the command - and its CPU, read from its scope's `cpu.stat` as `measure` does, or the - reaped children's CPU where it owns only a process group, to that line. - `QUEST_GATE_RECORD` stays the file a reader such as Kixel Code's job - runner owns. -- Ungated side: under T3 Code on Linux each agent session runs in a - systemd scope of its own, and gate scopes run outside it, so its - `cpu.stat` counts the harness plus every ungated command. Count only a - cgroup that holds the session alone: a terminal's shared scope would - charge the user's other work to it, so record null there, as on macOS. -- Calls that overlap, in parallel or in the background, are reported as - one group with their summed CPU. Sub-agents share their parent's scope, - so match calls by tool-use ID. -- Codex registers only a `PreToolUse` hook today; use its post-call event - if its hooks offer one, and record null otherwise. -- Report ranked lists, gated commands by least CPU and ungated calls by - most, with no threshold: the mistake audit judges what is a class. - -Baseline, 2026-11-01 to 11-15: - -- 32.8k gated calls on three hosts. `quest gate quest review`: 591, all - from Claude, at 2.2 CPU-seconds and 0.03 cores over a 58-second median - run. It peaked at 126 on 11-12, before - [moq-dev/quest#673](https://github.com/moq-dev/quest/pull/673) put model - calls outside the gate, and fell to 5 on 11-15; 27 of the 29 fedora - sessions that gated a review after 11-14 had loaded the older rule. -- 20% to 22% of 17.6k gate scopes over 3 days used under 5 CPU-seconds, - together 0.2% of gated CPU. Most were usually heavy commands that were - cheap that time, such as one `cargo test`; always-cheap tools, such as - one vitest file, `bun x`, or `cargo fmt`, were about 1% to 2% of calls. -- `nix build` shows about 1 CPU-second inside its scope, since - `nix-daemon` builds outside it; count it apart rather than as a cheap - command. -- Ungated CPU: not measured, the gap this quest fills. - -## Related - -- [Mistake audit](/quest/m0/mistake-audit.md) - reads these records as a - class each week -- [Remove the gate's quick flag](/quest/m0/gate-quick.md) - the same rule - change leaves the flag no use diff --git a/quest/m0/gate-quick.md b/quest/m0/gate-quick.md index 723ad892..cc189157 100644 --- a/quest/m0/gate-quick.md +++ b/quest/m0/gate-quick.md @@ -17,8 +17,3 @@ heavy commands, letting them start after 60 seconds whatever the load: `just fix` 41 times, `just check` 18, and `cargo build` 18. No repository script passes it outside Quest's own lifecycle check. Until this lands, `quest gate --help` still offers it for formatters. - -## Related - -- [Gating records](/quest/m0/benchmark/gating.md) - audits what agents - gate under the same rule diff --git a/quest/m0/mistake-audit.md b/quest/m0/mistake-audit.md index 01aaf366..f140130a 100644 --- a/quest/m0/mistake-audit.md +++ b/quest/m0/mistake-audit.md @@ -51,6 +51,14 @@ Agent's choices, for the implementer to revisit. Each run: merged fix's bullet that opens with the culprit's link, but a culprit can still name a PR that left the work to a planned follow-up or kept older behavior. +- Read both kinds of gating mistake from each record's `gated` and + `shell` lists: gated runs ranked by least CPU, and shell groups by most, + with no threshold, judging each command's class rather than one cheap + run of a usually heavy command. Count `nix build` apart, since + `nix-daemon` builds outside its gate's scope. A `shell` group's `cpu` is + null outside a cgroup that holds the session alone, as in a terminal, so + say how many groups were unmeasured. Records only hold calls from hosts + whose hooks `quest setup` updated since that PR. - `quest messages` returns every human turn as a `kind: "message"` entry, beside the `kind: "answer"` entries tallied below, so set aside machine-written ones yourself: bare skill invocations, pasted relays and logs, accepted @@ -114,8 +122,13 @@ Check first next run: [moq-dev/quest#652](https://github.com/moq-dev/quest/pull/652) lands. - Gating mistakes, after `quest context` said on 2026-11-15 to gate by CPU, not run time: `quest gate quest review` ran 591 times in the 14 - days before, 126 of them on 11-12. Count it from transcripts until - [gating records](/quest/m0/benchmark/gating.md) land, then from them. + days before, 126 of them on 11-12, at 2.2 CPU-seconds and 0.03 cores + over a 58-second median run; 20% to 22% of gate scopes over 3 days used + under 5 CPU-seconds, together 0.2% of gated CPU, most of them usually + heavy commands that were cheap that time. Ungated CPU was unmeasured + before + [moq-dev/quest#747](https://github.com/moq-dev/quest/pull/747); count + both from the records once it lands, from transcripts for earlier days. - Whether moq-dev/moq and moq-dev/moq.pro planned the runtime follow-ups of moq-dev/quest's 2026-11-11 run PR, and whether their escapes recurred. diff --git a/src/cli/calls.ts b/src/cli/calls.ts new file mode 100644 index 00000000..4a94284d --- /dev/null +++ b/src/cli/calls.ts @@ -0,0 +1,227 @@ +// The shell calls an agent session runs, sampled by Quest's tool hooks around each one: the CPU counter of the +// session's cgroup, where that cgroup holds the session alone. A session's record groups the calls that overlapped +// and charges each group what the cgroup used meanwhile, which `quest gate`'s own scopes, outside it, never add to. +import { readFileSync } from "node:fs"; +import { join } from "node:path/posix"; + +import { type ShellCall, type ShellGroup, commandLine } from "../core"; +import { cgroupUsage, members } from "./supervise"; +import { appendLog, logDirectory, readLog } from "./waits"; + +/** The session's cgroup and its CPU counter, in microseconds, when a hook sampled them. */ +export interface Sample { + at: string; + /** The cgroup's path, or null when it held more than the session, or on a host without one. */ + cgroup: string | null; + usage: number | null; +} + +/** Whose call a line is about: a Claude Code sub-agent's carries its agent ID beside its parent's session ID. */ +interface Caller { + harness: string; + session: string; + agent: string | null; +} + +/** + * One line of the call log: a shell call starting, one returning, with the background task a harness moved it to, + * or a Claude Code turn ending, with the background tasks still running then. + */ +export type CallLine = Sample & + ( + | (Caller & { kind: "start"; call: string; command: string }) + | (Caller & { kind: "end"; call: string; task: string | null }) + | { kind: "stop"; harness: string; session: string; running: string[] } + ); + +/** Where the call samples are logged. */ +export const callsDirectory = (env: NodeJS.ProcessEnv = process.env) => logDirectory("calls", env); + +/** Append `line` to today's call log. Throws when it cannot write. */ +export function logCall(line: CallLine, env: NodeJS.ProcessEnv = process.env, now = Date.now()): void { + appendLog(callsDirectory(env), line, now, "shell call"); +} + +/** A start line for `command`, cut as a record keeps it. */ +export function startLine(caller: Caller, call: string, command: string, sample: Sample): CallLine { + return { kind: "start", ...caller, call, command: commandLine(command), ...sample }; +} + +/** + * The call lines logged on this host from the UTC day before `since`'s on, so a call that started before it still + * has its start, failing on a line it cannot read. + */ +export function readCalls(since: number, env: NodeJS.ProcessEnv = process.env): CallLine[] { + return readLog(callsDirectory(env), since - 86_400_000, parseCall, "a shell call"); +} + +function parseCall(line: string): CallLine | null { + let value: Record; + try { + value = JSON.parse(line) as Record; + } catch { + return null; + } + const text = (v: unknown) => typeof v === "string" && v !== ""; + const sampled = + typeof value?.at === "string" && + !Number.isNaN(Date.parse(value.at)) && + (value.cgroup === null || text(value.cgroup)) && + (value.usage === null || (typeof value.usage === "number" && value.usage >= 0)) && + text(value.harness) && + text(value.session); + if (!sampled) return null; + const caller = (value.agent === null || text(value.agent)) && text(value.call); + switch (value.kind) { + case "start": + return caller && typeof value.command === "string" ? (value as unknown as CallLine) : null; + case "end": + return caller && (value.task === null || text(value.task)) ? (value as unknown as CallLine) : null; + case "stop": + return Array.isArray(value.running) && value.running.every(text) ? (value as unknown as CallLine) : null; + default: + return null; + } +} + +/** Where `sessionSample` reads processes and cgroups, and which process the hook and its harness are. */ +export interface Host { + proc: string; + cgroups: string; + self: number; + /** The harness's process, where it names it, as Claude Code does in `CLAUDE_PID`. */ + harness?: number; +} + +/** + * The cgroup holding this process and its CPU counter now, or nulls where it holds more than the hook's session. + * Such a cgroup holds one tree of processes, whose top is the harness: the hook's topmost ancestor in the cgroup, from + * which every other branch hangs, as when an app starts each agent session in a scope of its own. A terminal's scope + * fails, since the harness hangs below the terminal's shell, as does one shared with other work. A process whose + * parent left, as a daemonized one's does, is the session's leftover and counts. + */ +export function sessionSample( + now: number, + host: Host = { proc: "/proc", cgroups: "/sys/fs/cgroup", self: process.pid }, +): Sample { + const at = new Date(now).toISOString(); + const none = { at, cgroup: null, usage: null }; + if (process.platform !== "linux") return none; + const path = readFileSync(join(host.proc, String(host.self), "cgroup"), "utf8") + .split("\n") + .find((line) => line.startsWith("0::")) + ?.slice(3); + if (path === undefined) return none; + const group = join(host.cgroups, path); + const inside = new Set(members(group)); + const parents = new Map(); + const parent = (pid: number) => { + if (!parents.has(pid)) parents.set(pid, parentOf(host.proc, pid)); + return parents.get(pid); + }; + const chain = new Set(); + let top = host.self; + for (let pid: number | undefined = host.self; pid !== undefined && inside.has(pid); pid = parent(pid)) { + chain.add(pid); + top = pid; + } + if (!chain.has(host.self) || (host.harness !== undefined && host.harness !== top)) return none; + for (const pid of inside) { + if (chain.has(pid)) continue; + // The chain member this process hangs from, or undefined for a tree whose root's parent left the cgroup. + let at: number | undefined = parent(pid); + while (at !== undefined && inside.has(at) && !chain.has(at)) at = parent(at); + if (at !== undefined && chain.has(at) && at !== top) return none; + } + const usage = cgroupUsage(group); + return usage === null ? none : { at, cgroup: path, usage }; +} + +/** Field 4 of /proc//stat, or undefined once the process is gone. */ +function parentOf(proc: string, pid: number): number | undefined { + let stat: string; + try { + stat = readFileSync(join(proc, String(pid), "stat"), "utf8"); + } catch (error) { + if (["ENOENT", "ESRCH"].includes((error as NodeJS.ErrnoException).code ?? "")) return undefined; + throw error; + } + return Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[1]); +} + +/** A shell group, and the sessions, as `harness\0session` with a Claude sub-agent's agent ID, that ran its calls. */ +export interface OwnedGroup { + group: ShellGroup; + sessions: Set; + /** Its last call's end, epoch ms. */ + end: number; +} + +type Start = Extract; + +/** + * The calls in `lines`, grouped where they overlapped in one cgroup, each group charged what its cgroup used from its + * first call's start to its last call's end. A call that never returned, as one denied or interrupted, is left out. A + * call a Claude Code session moved to the background runs until the first turn end that no longer lists its task, + * else until its cgroup's last sample. A call sampled outside a counted cgroup is a group of its own, without CPU. + */ +export function shellGroups(lines: CallLine[]): OwnedGroup[] { + const callKey = (l: { harness: string; session: string; call: string }) => `${l.harness}\0${l.session}\0${l.call}`; + const ends = new Map>(); + const stops = new Map[]>(); + const last = new Map(); + const ms = (s: Sample) => Date.parse(s.at); + for (const line of lines) { + if (line.kind === "end") ends.set(callKey(line), line); + if (line.kind === "stop") { + const key = `${line.harness}\0${line.session}`; + stops.set(key, [...(stops.get(key) ?? []), line]); + } + if (line.cgroup !== null && ms(line) >= ms(last.get(line.cgroup) ?? line)) last.set(line.cgroup, line); + } + const calls: { start: Start; end: Sample }[] = []; + for (const start of lines) { + if (start.kind !== "start") continue; + const returned = ends.get(callKey(start)); + if (!returned || ms(returned) < ms(start)) continue; + let end: Sample = returned; + if (returned.task !== null) { + const finished = (stops.get(`${start.harness}\0${start.session}`) ?? []) + .filter((stop) => ms(stop) >= ms(returned) && !stop.running.includes(returned.task!)) + .sort((a, b) => ms(a) - ms(b))[0]; + end = finished ?? (start.cgroup === null ? returned : (last.get(start.cgroup) ?? returned)); + } + calls.push({ start, end }); + } + calls.sort((a, b) => ms(a.start) - ms(b.start)); + const groups: { calls: typeof calls; end: Sample }[] = []; + const open = new Map(); + for (const call of calls) { + const current = call.start.cgroup === null ? undefined : open.get(call.start.cgroup); + if (current && ms(call.start) <= ms(current.end)) { + current.calls.push(call); + if (ms(call.end) > ms(current.end)) current.end = call.end; + continue; + } + const group = { calls: [call], end: call.end }; + groups.push(group); + if (call.start.cgroup !== null) open.set(call.start.cgroup, group); + } + return groups.map(({ calls: grouped, end }) => { + const first = grouped[0]!.start; + const counted = + first.cgroup !== null && + end.cgroup === first.cgroup && + first.usage !== null && + end.usage !== null && + end.usage >= first.usage; + return { + group: { + calls: grouped.map((c): ShellCall => ({ command: c.start.command, start: c.start.at, end: c.end.at })), + cpu: counted ? Math.round((end.usage! - first.usage!) / 1000) : null, + }, + sessions: new Set(grouped.map((c) => `${c.start.harness}\0${c.start.agent ?? c.start.session}`)), + end: ms(end), + }; + }); +} diff --git a/src/cli/gate.ts b/src/cli/gate.ts index a3f0a173..de1773aa 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -24,8 +24,9 @@ import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; import { stubbed } from "./job"; +import { commandLine } from "../core"; import { envSessions, logWait } from "./waits"; -import { questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; +import { cgroupUsage, measuredRun, questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; /** * One admitted command: a unique id, the wall-clock time it was admitted, and the process ID of @@ -436,17 +437,8 @@ export function fileHost(directory: string, now = Date.now): Host { const group = groups.get(unit) ?? scopeGroup(unit); if (group === undefined) return undefined; groups.set(unit, group); - let stat: string; - try { - stat = readFileSync(join(group, "cpu.stat"), "utf8"); - } catch (error) { - // The scope ended, and systemd removed its cgroup. - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; - throw error; - } - const usage = stat.match(/^usage_usec (\d+)$/m)?.[1]; - if (usage === undefined) throw new Error(`gate cannot read the CPU use of ${unit}`); - return Number(usage); + // Null once the scope ended, and systemd removed its cgroup. + return cgroupUsage(group) ?? undefined; }, }; } @@ -740,21 +732,29 @@ export const inside: Owner = plain(true); * `enclosing`, so the enclosing scope keeps its own weight for the session's other work, its * owner's kill stops the gate's scope too, and the gate forwards that owner's signals to it. */ -const background = - (unit: string, enclosing: string | null): Owner => - (command, env, cancel) => - systemdScopes.run(unit, process.cwd(), command, { +function background(unit: string, enclosing: string | null): Measuring { + let used: number | null = null; + const run: Owner = async (command, env, cancel) => { + const { status, cpu } = await measuredRun(unit, process.cwd(), command, { env, cancel, within: enclosing, slice: "background.slice", }); + used = cpu === null ? null : Math.round(cpu / 1000); + return status; + }; + return Object.assign(run, { cpu: () => used }); +} + +/** An owner that also reports the CPU milliseconds its last run used, or null when it could not read them. */ +export type Measuring = Owner & { cpu(): number | null }; /** Run the command in background scope `unit`, a new name, outside any Quest scope (see `background`). */ -export const scope = (unit: string): Owner => background(unit, null); +export const scope = (unit: string): Measuring => background(unit, null); /** Run the command in background scope `unit`, a new name, bound to the Quest scope `enclosing` (see `background`). */ -export const bound = (unit: string, enclosing: string): Owner => background(unit, enclosing); +export const bound = (unit: string, enclosing: string): Measuring => background(unit, enclosing); /** * On Windows, run the command under a stub in a kill-on-close Job Object that forbids breakaway @@ -849,7 +849,7 @@ const TOKEN = 3; * like macOS, without a systemd user manager. Windows owns them through a Job Object. A systemd * scope also lets the gate measure its command's CPU use (see `measure`), so `unit` names it. */ -export function owner(env = process.env, cgroup = "/proc/self/cgroup"): { run: Owner; unit?: string } { +export function owner(env = process.env, cgroup = "/proc/self/cgroup"): { run: Owner | Measuring; unit?: string } { // A Quest scope around this process holds the command too, since it shares the cgroup or an ancestor. const enclosing = process.platform === "linux" ? questScope(cgroup) : undefined; if (env[ADMITTED] === "1") return { run: env[GROUP] === "1" || enclosing ? inside : direct }; @@ -910,6 +910,8 @@ interface Observed { admitted?: number; /** The systemd scope the command ran in, whose journal lines hold its CPU time and peak memory. */ unit?: string; + /** The CPU milliseconds the command used, where its owner measured them. */ + cpu?: number | null; } /** @@ -940,20 +942,19 @@ export async function gate(command: string[], options: GateOptions = {}): Promis status = 1; } if (waits) { - const ended = Date.now(); const iso = (ms: number) => new Date(ms).toISOString(); try { - // A run cancelled before admission waited in the queue until it ended. - const admitted = observed.admitted ?? ended; logWait( - { kind: "gate-queue", start: iso(started), end: iso(admitted), until: null }, + { + kind: "gate", + command: commandLine(command.join(" ")), + start: iso(started), + admitted: observed.admitted === undefined ? null : iso(observed.admitted), + end: iso(Date.now()), + cpu: observed.cpu ?? null, + }, { cwd: directory }, ); - if (observed.admitted !== undefined) - logWait( - { kind: "gate-run", start: iso(admitted), end: iso(ended), until: null }, - { cwd: directory }, - ); } catch (error) { // A cancelled run exits as it would have, logged or not. if (cancel.signal.aborted) process.stderr.write(`quest gate: ${message(error)}\n`); @@ -1053,6 +1054,7 @@ async function gated( result = await run(command, env, cancel); } finally { stop.abort(); + if ("cpu" in run) observed.cpu = run.cpu(); } const failed = await measuring; if (cancel.signal.aborted) return cancelled(cancel.signal); diff --git a/src/cli/hooks.ts b/src/cli/hooks.ts index 0301e4ef..0d724d61 100644 --- a/src/cli/hooks.ts +++ b/src/cli/hooks.ts @@ -15,6 +15,7 @@ import { stubFindings } from "./setup"; import { CONTEXT } from "./skills"; import { HOOK_EVENTS } from "./worktree-hook"; import { queue } from "./send"; +import { type Sample, logCall, sessionSample, startLine } from "./calls"; /** The command the rule hooks run. */ export const HOOK_COMMAND = "quest hook"; @@ -44,14 +45,17 @@ const SESSION = "startup|resume|clear|compact"; * The entries `quest setup` installs for Claude Code. SessionStart prints the * rules; SubagentStart gives each subagent the same rules, since it starts * without the session's. PreToolUse blocks the calls that break a rule a - * program can check. Stop and SubagentStop refuse to end a turn while a gate - * the agent started still runs in the background; once a session's turn ends, - * Stop queues its record to send. + * program can check. PreToolUse and the PostToolUse events also sample each + * shell call's CPU (see calls.ts). Stop and SubagentStop refuse to end a turn + * while a gate the agent started still runs in the background; once a + * session's turn ends, Stop queues its record to send. */ export const CLAUDE_HOOKS: readonly HookEntry[] = [ { event: "SessionStart", matcher: SESSION }, { event: "SubagentStart" }, { event: "PreToolUse", matcher: "Bash|Write|Edit|MultiEdit" }, + { event: "PostToolUse", matcher: "Bash" }, + { event: "PostToolUseFailure", matcher: "Bash" }, { event: "Stop" }, { event: "SubagentStop" }, ]; @@ -64,6 +68,7 @@ export const CLAUDE_HOOKS: readonly HookEntry[] = [ export const CODEX_HOOKS: readonly HookEntry[] = [ { event: "SessionStart", matcher: SESSION, settings: { additionalContextLimit: CONTEXT_LIMIT } }, { event: "PreToolUse", matcher: "Bash|apply_patch" }, + { event: "PostToolUse", matcher: "Bash" }, { event: "Stop" }, ]; @@ -279,13 +284,21 @@ interface HookInput { session_id?: string; /** Null for a session that keeps no transcript. */ transcript_path?: string | null; - /** Codex's Stop names its turn; Claude Code's does not. */ + /** Codex names its turn in every event; Claude Code in none. */ turn_id?: string; + /** The tool call a PreToolUse or PostToolUse event is about. */ + tool_use_id?: string; + /** Claude Code: the sub-agent whose call it is. */ + agent_id?: string; + /** What the tool returned; Claude Code's Bash names the background task it moved the command to. */ + tool_response?: unknown; } /** What `hook` reads beyond its input. */ export interface HookDependencies { env: NodeJS.ProcessEnv; + /** The session's cgroup and CPU counter now (see `sessionSample`). */ + sample?: () => Sample; } /** What a Stop or SubagentStop refusal tells the agent, after the gates it names. */ @@ -298,8 +311,9 @@ const BLOCK_ON_GATES = * Quest's reference line. SessionStart and SubagentStart add `quest context` * to the session's context, so its agents never run the command themselves. * PreToolUse denies a tool call that breaks a rule `broken` checks, giving the - * rule as the reason. Stop and SubagentStop refuse to end the turn while a gate - * the agent started runs in the background (see `runningGates`). + * rule as the reason; it and the PostToolUse events log a shell call they let + * through (see calls.ts). Stop and SubagentStop refuse to end the turn while a + * gate the agent started runs in the background (see `runningGates`). */ export function hook(input: unknown, deps: HookDependencies = { env: process.env }): string { const value = input as Partial | null; @@ -327,7 +341,14 @@ export function hook(input: unknown, deps: HookDependencies = { env: process.env throw new Error("expected a PreToolUse hook's JSON input with tool_name and tool_input"); if (!usesQuest(value.cwd)) return ""; const rules = broken({ tool: value.tool_name, input: value.tool_input, cwd: value.cwd }); - if (rules.length === 0) return ""; + if (rules.length === 0) { + if (value.tool_name === "Bash" && typeof value.tool_input.command === "string") + logCall( + startLine(caller(value), callId(value), value.tool_input.command, sample(deps, value)), + deps.env, + ); + return ""; + } const reason = rules.join("\n"); return JSON.stringify({ hookSpecificOutput: { @@ -337,6 +358,14 @@ export function hook(input: unknown, deps: HookDependencies = { env: process.env }, }); } + case "PostToolUse": + case "PostToolUseFailure": { + if (value.tool_name !== "Bash" || !usesQuest(value.cwd)) return ""; + const response = value.tool_response as { backgroundTaskId?: unknown } | null | undefined; + const task = typeof response?.backgroundTaskId === "string" ? response.backgroundTaskId : null; + logCall({ kind: "end", ...caller(value), call: callId(value), task, ...sample(deps, value) }, deps.env); + return ""; + } case "Stop": case "SubagentStop": { // Codex runs no background gates, so its Stop only queues the record. @@ -348,6 +377,20 @@ export function hook(input: unknown, deps: HookDependencies = { env: process.env return stopped(value, "codex", deps); if (!Array.isArray((value as StopInput).background_tasks)) throw new Error(`expected a ${event} hook's JSON input with background_tasks; upgrade Claude Code`); + // Closes the shell calls Claude Code moved to the background (see `shellGroups`). + if (typeof value.session_id === "string" && usesQuest(value.cwd)) + logCall( + { + kind: "stop", + harness: "claude", + session: value.session_id, + running: (value as StopInput) + .background_tasks!.filter((task) => task.type === "shell") + .map((task) => task.id), + ...sample(deps, value), + }, + deps.env, + ); const gates = runningGates(value as StopInput); if (gates.length === 0) return event === "Stop" ? stopped(value, "claude", deps) : ""; const named = gates.map((command) => `- ${command}`).join("\n"); @@ -361,6 +404,35 @@ export function hook(input: unknown, deps: HookDependencies = { env: process.env } } +/** Whose tool call `value` is about: Codex names its turn in every event, and Claude Code a sub-agent's ID. */ +function caller(value: Partial): { harness: string; session: string; agent: string | null } { + if (typeof value.session_id !== "string") + throw new Error(`expected a ${value.hook_event_name} hook's JSON input with session_id`); + return { + harness: typeof value.turn_id === "string" ? "codex" : "claude", + session: value.session_id, + agent: typeof value.agent_id === "string" ? value.agent_id : null, + }; +} + +function callId(value: Partial): string { + if (typeof value.tool_use_id !== "string" || value.tool_use_id === "") + throw new Error(`expected a ${value.hook_event_name} hook's JSON input with tool_use_id; upgrade the agent`); + return value.tool_use_id; +} + +/** The session's sample now; Claude Code names its own process, which an agent it starts inherits. */ +function sample(deps: HookDependencies, value: Partial): Sample { + if (deps.sample) return deps.sample(); + const harness = typeof value.turn_id === "string" ? NaN : Number(deps.env.CLAUDE_PID); + return sessionSample(Date.now(), { + proc: "/proc", + cgroups: "/sys/fs/cgroup", + self: process.pid, + ...(Number.isSafeInteger(harness) && harness > 0 ? { harness } : {}), + }); +} + /** * Queue the record of the session whose turn ended for the user's sender. A Claude Code sub-agent is sent with its * parent. Says what the person has not seen yet: why nothing is sent, that no sender runs, or a sender's failure. diff --git a/src/cli/main.ts b/src/cli/main.ts index 5dd21bb7..2424f7e3 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -90,6 +90,7 @@ import { import { CHECKED_CONTEXT, USER_ASKED, questRoot } from "./rules"; import { messageLines, messages } from "./messages"; import { logWait, pullWaitKind, readWaits } from "./waits"; +import { readCalls } from "./calls"; import { CHECKOUT_CONFIG, SEND_INTERVAL, @@ -1273,6 +1274,7 @@ export function main(argv: string[], out: Output): number | Promise { github: () => gitHubDependencies(warn), sessions: (since) => runTokenMax(since, []), waits: (since) => readWaits(since), + calls: (since) => readCalls(since), ask, fetch: globalThis.fetch, sleep: (ms) => new Promise((done) => setTimeout(done, ms)), @@ -1871,7 +1873,8 @@ export function main(argv: string[], out: Output): number | Promise { "calls there that break a rule a program can check: em dashes an agent writes, a push without an " + "explicit refspec, a force-push or rebase, a GitHub post without its `(Written by ...)` footer, and " + "scratch files in /tmp. Stop and SubagentStop refuse to end an agent's turn while a `quest gate` " + - "command it started runs in the background. Each runs `quest hook`, so `quest` must be on the PATH the agent starts " + + "command it started runs in the background. PreToolUse and the PostToolUse hooks also sample each " + + "shell call's CPU for the session's record. Each runs `quest hook`, so `quest` must be on the PATH the agent starts " + "with, and Codex asks to trust its hooks once through /hooks.\n\n" + "With --worktrees, also installs Claude Code's " + WORKTREE_HOOKS.map((h) => h.event).join(", ") + @@ -1906,7 +1909,10 @@ export function main(argv: string[], out: Output): number | Promise { "or a commit message or new GitHub post that adds an em dash outside a `>` quote line, a `git push` " + "whose refspecs lack a `:`, a force-push, a `git rebase` or `git pull --rebase`, a new GitHub post " + "whose body lacks a `(Written by ...)` line, or a file written under /tmp. A force-push or rebase the " + - `user asked for passes with the \`${USER_ASKED}\` prefix. Stop and SubagentStop refuse to end the ` + + `user asked for passes with the \`${USER_ASKED}\` prefix. Around each shell call it lets through, ` + + "PreToolUse and PostToolUse (or Claude Code's PostToolUseFailure) log the session's cgroup CPU counter " + + "under $XDG_STATE_HOME/quest/calls, where that cgroup holds the session alone, for its record. " + + "Stop and SubagentStop refuse to end the " + "turn, naming each command, while a `quest gate` command the ending agent started runs in Claude " + "Code's background: one whose background ID its transcript holds. Otherwise Claude Code's and Codex's Stop " + `queue the session's record for the endpoint the checkout's ${CHECKOUT_CONFIG} names, for quest records ` + diff --git a/src/cli/records.ts b/src/cli/records.ts index ff148675..985f9952 100644 --- a/src/cli/records.ts +++ b/src/cli/records.ts @@ -1,5 +1,6 @@ // Each session's record, as `quest usage` prints it and the stop hook sends it: its PRs, role, spend, and waits. -import { type Role, type SessionRecord, type SessionWait, type TimeSplit, branchQuest } from "../core"; +import { type GatedRun, type Role, type SessionRecord, type SessionWait, type TimeSplit, branchQuest } from "../core"; +import { type CallLine, shellGroups } from "./calls"; import { type TranscriptFacts, transcriptFacts } from "./transcript"; import type { Pull, Session } from "./usage"; import type { LoggedWait } from "./waits"; @@ -38,8 +39,10 @@ export interface RecordInput { /** Session IDs a PR's review record names as reviewers, and as its risk classifier. */ reviewers: Set; classifiers: Set; - /** The waits logged on this host, which only its own sessions (host null) are given. */ + /** The waits and gate runs logged on this host, which only its own sessions (host null) are given. */ waits: LoggedWait[]; + /** The shell calls sampled on this host, likewise. */ + calls: CallLine[]; /** The sessions to build records for; by default, those on a PR. Every session still counts for lineage. */ emit?: Set; /** The window's start, epoch ms: a person's wait that ended before it, like a logged one, is left out. */ @@ -126,6 +129,7 @@ export function buildRecords(input: RecordInput): Map { }; const waits = assignWaits(named, input.waits, parentOf, input.since); + const groups = shellGroups(input.calls).filter((g) => g.end >= input.since); const records = new Map(); for (const s of input.sessions) { const { pulls, via } = pullsOf(s); @@ -141,13 +145,18 @@ export function buildRecords(input: RecordInput): Map { end: iso(q.end), until: null, })); + const logged = waits.get(s) ?? []; const timed: SessionWait[] | null = local === null ? null - : [ - ...(waits.get(s) ?? []).map(({ kind, start, end, until }) => ({ kind, start, end, until })), - ...person, - ].sort((a, b) => a.start.localeCompare(b.start)); + : [...logged.flatMap(timedWaits).filter((w) => Date.parse(w.end) >= input.since), ...person].sort( + (a, b) => a.start.localeCompare(b.start), + ); + const gated: GatedRun[] = logged.flatMap((w) => + w.kind === "gate" + ? [{ command: w.command, start: w.start, admitted: w.admitted, end: w.end, cpu: w.cpu }] + : [], + ); records.set(s, { harness: s.harness, session: s.id, @@ -172,12 +181,26 @@ export function buildRecords(input: RecordInput): Map { start: iso(s.start), end: iso(s.end), waits: timed, + gated: s.host === null ? gated : null, + shell: + s.host === null && s.id !== null + ? groups.filter((g) => g.sessions.has(id(s.harness, s.id!))).map((g) => g.group) + : s.host === null + ? [] + : null, split: timed === null || !input.split ? null : input.split(s, timed, input.since), }); } return records; } +/** The waits a logged line times: a gate run's queue until its admission, and its run from then. */ +function timedWaits(w: LoggedWait): SessionWait[] { + if (w.kind !== "gate") return [{ kind: w.kind, start: w.start, end: w.end, until: w.until }]; + const queued: SessionWait = { kind: "gate-queue", start: w.start, end: w.admitted ?? w.end, until: null }; + return w.admitted === null ? [queued] : [queued, { kind: "gate-run", start: w.admitted, end: w.end, until: null }]; +} + /** * A local session's transcript facts, or null when they cannot be read: a file deleted since token-max read it, or * one with lines token-max could not read either, whose cost is already a lower bound. Any other failure is a shape diff --git a/src/cli/send.ts b/src/cli/send.ts index e6d70dd9..57c6e8d2 100644 --- a/src/cli/send.ts +++ b/src/cli/send.ts @@ -18,6 +18,7 @@ import { questRoot } from "./rules"; import { type Labeled, type SplitDependencies, type Steps, labelSession, readSteps, splitOf } from "./split"; import { type Pull, type Session, fixBreakages, fold, key, readPull, sessionRecords, updatedPulls } from "./usage"; import { type LoggedWait } from "./waits"; +import type { CallLine } from "./calls"; /** The checkout's file naming its record endpoint, as `{"records": "https://..."}`. */ export const CHECKOUT_CONFIG = ".quest.json"; @@ -238,6 +239,7 @@ export interface SendDependencies { /** token-max's sessions on this host since an instant, or null when it is not installed. */ sessions(since: Date): Session[] | null; waits(since: number): LoggedWait[]; + calls(since: number): CallLine[]; /** Asks the models that split each session's time. */ ask: SplitDependencies["ask"]; fetch: typeof globalThis.fetch; @@ -351,6 +353,7 @@ async function sendOnce(deps: SendDependencies, dir: string): Promise { const { kept } = fold(exported); const byId = new Map(kept.flatMap((s) => (s.id === null ? [] : [[`${s.harness}\0${s.id}`, s] as const]))); const waits = deps.waits(since); + const calls = deps.calls(since); const client = new GitHub(deps.github()); const sent: Sent[] = []; const failures: string[] = []; @@ -463,6 +466,7 @@ async function sendOnce(deps: SendDependencies, dir: string): Promise { evidence: exported, pulls, waits, + calls, emit, since, split: (s, waits, from) => { diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index 7bde5a57..8984946d 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -102,12 +102,20 @@ export const UNSUPPORTED = * The shell proves it runs in the scope's own v2 cgroup, then waits for the supervisor before it * executes the command. A supervisor that died never acknowledges, so the command never starts. */ -const handshake = +const handshake = (hold: boolean) => "IFS=: read -r hierarchy controllers path &3; ' + "read -r admitted <&4 || exit 125; " + - 'exec 3>&- 4<&-; exec "$@"'; + `exec 3>&- 4<&-${hold ? " 5<&-" : ""}; exec "$@"`; + +/** + * Keeps the scope's cgroup until the supervisor has read what it used: systemd removes an empty scope's cgroup as + * the command exits, sometimes before the supervisor sees the exit. A process that reads descriptor 5 until the + * supervisor closes it, or dies, stays in the cgroup; double-forked, it is no child the command could wait for. + */ +const HOLD = "( (read -r _ <&5) 3>&- 4<&- /dev/null 2>&1 & ); "; function populated(group: string): boolean { try { @@ -252,6 +260,12 @@ async function stopBound(unit: string): Promise { for (const bound of boundTo(unit)) await systemdScopes.stop(bound); } +/** What a measured run ended with: its exit status, and the CPU microseconds its scope used, when they could be read. */ +export interface Measured { + status: number; + cpu: number | null; +} + export const systemdScopes: Scopes = { check() { if (process.platform !== "linux" || !killable()) throw new Error(UNSUPPORTED); @@ -286,41 +300,65 @@ export const systemdScopes: Scopes = { }, run(unit, cwd, command, options = {}) { - // Once the enclosing owner kills everything it holds, systemd kills this scope too. - const enclosing = options.within === undefined ? questScope()?.unit : options.within; - const bound = enclosing ? [`--property=BindsTo=${enclosing}`, "--property=KillSignal=SIGKILL"] : []; - return contain( - [ - "systemd-run", - "--user", - "--scope", - "--quiet", - "--collect", - `--unit=${unit}`, - ...bound, - ...(options.slice ? [`--slice=${options.slice}`] : []), - "--expand-environment=no", - "--", - "/bin/sh", - "-c", - handshake, - "quest-worktree", - ], - (path) => path.endsWith(`/${unit}`), - { - name: unit, - unsupported: UNSUPPORTED, - launcher: "systemd-run", - emptied: () => stopBound(unit), - bound: Boolean(enclosing), - }, - cwd, - command, - options, - ); + return scoped(unit, cwd, command, options, false).then(({ status }) => status); }, }; +/** Like `systemdScopes.run`, also reading the CPU its scope used once the command exits. */ +export function measuredRun(unit: string, cwd: string, command: string[], options: RunOptions = {}): Promise { + return scoped(unit, cwd, command, options, true); +} + +function scoped(unit: string, cwd: string, command: string[], options: RunOptions, measure: boolean) { + // Once the enclosing owner kills everything it holds, systemd kills this scope too. + const enclosing = options.within === undefined ? questScope()?.unit : options.within; + const bound = enclosing ? [`--property=BindsTo=${enclosing}`, "--property=KillSignal=SIGKILL"] : []; + return contain( + [ + "systemd-run", + "--user", + "--scope", + "--quiet", + "--collect", + `--unit=${unit}`, + ...bound, + ...(options.slice ? [`--slice=${options.slice}`] : []), + "--expand-environment=no", + "--", + "/bin/sh", + "-c", + handshake(measure), + "quest-worktree", + ], + (path) => path.endsWith(`/${unit}`), + { + name: unit, + unsupported: UNSUPPORTED, + launcher: "systemd-run", + emptied: () => stopBound(unit), + bound: Boolean(enclosing), + measure, + }, + cwd, + command, + options, + ); +} + +/** The CPU microseconds the cgroup used, or null once it is gone. */ +export function cgroupUsage(group: string): number | null { + let stat: string; + try { + stat = readFileSync(join(group, "cpu.stat"), "utf8"); + } catch (error) { + if (["ENOENT", "ENODEV"].includes((error as NodeJS.ErrnoException).code ?? "")) return null; + throw error; + } + const usage = /^usage_usec (\d+)$/m.exec(stat)?.[1]; + if (usage === undefined) throw new Error(`cannot read the CPU use of ${group}`); + return Number(usage); +} + /** How `contain` names, signals, and shares its run's cgroup. */ interface Containment { /** The run's scope or cgroup. */ @@ -336,6 +374,8 @@ interface Containment { * signal this process receives, which may be a copy of that owner's, leaves the kill to it. */ bound: boolean; + /** Whether to read the CPU the cgroup used once the command exits, holding the cgroup until then (see `HOLD`). */ + measure: boolean; } /** @@ -350,7 +390,7 @@ function contain( cwd: string, command: string[], options: RunOptions, -): Promise { +): Promise { return new Promise((resolve, reject) => { const own = options.cancel ? undefined : interrupts(); const cancel = options.cancel ?? own!; @@ -358,11 +398,14 @@ function contain( const child = spawn(launch[0], [...launch.slice(1), ...command], { cwd, env: options.env, - stdio: [pipe, pipe, pipe, "pipe", "pipe"], + stdio: [pipe, pipe, pipe, "pipe", "pipe", ...(describe.measure ? ["pipe" as const] : [])], }); options.spawned?.(child as ChildProcessWithoutNullStreams); const report = child.stdio[3] as Readable; const admission = (child.stdio as unknown as (Writable | null)[])[4]!; + // Closing it releases the process holding the cgroup. + const hold = (child.stdio as unknown as (Readable | null)[])[5]; + hold?.on("error", () => {}); let group: string | undefined; let admitted = false; let interrupted = 0; @@ -424,6 +467,7 @@ function contain( }); child.once("error", (error) => { + hold?.destroy(); cleanup(); reject( (error as NodeJS.ErrnoException).code === "ENOENT" @@ -432,6 +476,14 @@ function contain( ); }); child.once("exit", async (code, signal) => { + let cpu: number | null = null; + try { + // Read before anything else, while the holder keeps the cgroup; a run cancelled hard may have lost it. + if (describe.measure && group) cpu = cgroupUsage(group); + } catch (error) { + failure ??= error as Error; + } + hold?.destroy(); try { // Descendants that outlive the command are orphans of this run: stop them. if (group) { @@ -453,7 +505,7 @@ function contain( throw new Error( `${describe.unsupported}: ${describe.launcher} exited with ${code ?? signal} before the command started`, ); - resolve(interrupted || (code ?? 128 + constants.signals[signal!])); + resolve({ status: interrupted || (code ?? 128 + constants.signals[signal!]), cpu }); } catch (error) { reject(error); } finally { diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 09b632f3..cccb5a9c 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -40,6 +40,7 @@ import { type RecordInput, buildRecords } from "./records"; import { stateDirectory } from "./send"; import { keptSplit } from "./split"; import { type LoggedWait, readWaits } from "./waits"; +import { type CallLine, readCalls } from "./calls"; /** A session's tokens, and their cost at the model's API list price, which subscriptions do not bill. */ export interface Tokens { @@ -268,6 +269,8 @@ export interface UsageDependencies extends ClaimDependencies { sessions(since: Date, ssh: string[]): Session[] | null; /** The waits logged on this host that ended since an instant, epoch ms. */ waits(since: number): LoggedWait[]; + /** The shell calls sampled on this host around an instant, epoch ms (see `readCalls`). */ + calls(since: number): CallLine[]; /** A local session's time split from the labels the record sender kept, never asking a model. */ split: NonNullable; } @@ -630,6 +633,8 @@ export interface RecordSources { pulls: Pull[]; /** The waits logged on this host. */ waits: LoggedWait[]; + /** The shell calls sampled on this host. */ + calls: CallLine[]; /** The sessions to build records for; by default, those on a PR. */ emit?: Set; /** The window's start, epoch ms. */ @@ -640,7 +645,7 @@ export interface RecordSources { /** Each session's record, attributed to the PRs in the window, whose review records name their reviewers and classifier. */ export function sessionRecords(sources: RecordSources): Map { - const { repository, sessions, evidence, pulls, waits, emit, since, split } = sources; + const { repository, sessions, evidence, pulls, waits, calls, emit, since, split } = sources; // Only the reviewer sessions are read, so no PR author can stop every session's record with a body this version // cannot parse. const reviewers = new Set(pulls.flatMap((pull) => reviewSessions(pull.body))); @@ -652,6 +657,7 @@ export function sessionRecords(sources: RecordSources): Map readWaits(since), + calls: (since) => readCalls(since), split: (session, waits, since) => keptSplit(state, session, waits, since), }; } diff --git a/src/cli/waits.ts b/src/cli/waits.ts index 281a222e..d0cbd9b3 100644 --- a/src/cli/waits.ts +++ b/src/cli/waits.ts @@ -1,10 +1,11 @@ -// The waits `quest gate` and `quest wait` time for the agent session that ran them, kept on this host so a session's -// record can add them up. The harness names its session in each command's environment; nothing is logged outside one. +// The waits `quest gate` and `quest wait` time for the agent session that ran them, and the gate's runs, kept on this +// host so a session's record can add them up. The harness names its session in each command's environment; nothing +// is logged outside one. import { closeSync, constants, mkdirSync, openSync, readdirSync, readFileSync, rmSync, writeSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; -import { type SessionWait, WAIT_KINDS } from "../core"; +import { type GatedRun, type SessionWait, WAIT_KINDS } from "../core"; /** A session a harness names in a command's environment. */ export interface EnvSession { @@ -12,13 +13,16 @@ export interface EnvSession { session: string; } -/** One logged wait, with every session its environment named and the directory it ran in. */ -export interface LoggedWait extends SessionWait { +/** One outermost `quest gate` run, as the gate logs it; a record takes its queue and run waits from it. */ +export type GateLine = { kind: "gate" } & GatedRun; + +/** One logged wait or gate run, with every session its environment named and the directory it ran in. */ +export type LoggedWait = (SessionWait | GateLine) & { sessions: EnvSession[]; directory: string; -} +}; -/** Days of waits kept, beyond the longest session a record is expected to span. */ +/** Days of logs kept, beyond the longest session a record is expected to span. */ const KEEP_DAYS = 90; const DAY = 86_400_000; @@ -33,31 +37,24 @@ export function envSessions(env: NodeJS.ProcessEnv = process.env): EnvSession[] return found; } -/** Where the waits are logged, one file per UTC day. */ -export function waitsDirectory(env: NodeJS.ProcessEnv = process.env): string { - return join(env.XDG_STATE_HOME || join(homedir(), ".local", "state"), "quest", "waits"); +/** Where `quest`'s per-session logs on this host live, each one file per UTC day under its own directory. */ +export function logDirectory(name: string, env: NodeJS.ProcessEnv = process.env): string { + return join(env.XDG_STATE_HOME || join(homedir(), ".local", "state"), "quest", name); } +/** Where the waits are logged. */ +export const waitsDirectory = (env: NodeJS.ProcessEnv = process.env) => logDirectory("waits", env); + const day = (ms: number) => new Date(ms).toISOString().slice(0, 10); /** - * Append `wait` to today's log in one write, so concurrent writers leave only whole lines, when the environment - * names a session; drop days past `KEEP_DAYS`. Throws when it cannot write. + * Append `value` to today's file in `dir` in one write, so concurrent writers leave only whole lines, and drop + * days past `KEEP_DAYS`. Throws when it cannot write, naming what it logged. */ -export function logWait( - wait: SessionWait, - options: { env?: NodeJS.ProcessEnv; cwd?: string; now?: number } = {}, -): void { - const env = options.env ?? process.env; - const sessions = envSessions(env); - if (sessions.length === 0) return; - const now = options.now ?? Date.now(); - const dir = waitsDirectory(env); +export function appendLog(dir: string, value: unknown, now: number, what: string): void { try { mkdirSync(dir, { recursive: true, mode: 0o700 }); - const line = Buffer.from( - `${JSON.stringify({ ...wait, sessions, directory: options.cwd ?? process.cwd() } satisfies LoggedWait)}\n`, - ); + const line = Buffer.from(`${JSON.stringify(value)}\n`); const fd = openSync( join(dir, `${day(now)}.jsonl`), constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, @@ -74,13 +71,15 @@ export function logWait( if (/^\d{4}-\d\d-\d\d\.jsonl$/.test(name) && name.slice(0, 10) < oldest) rmSync(join(dir, name), { force: true }); } catch (error) { - throw new Error(`cannot log the wait in ${dir}: ${error instanceof Error ? error.message : String(error)}`); + throw new Error(`cannot log the ${what} in ${dir}: ${error instanceof Error ? error.message : String(error)}`); } } -/** The waits logged on this host that ended at or after `since`, failing on a line it cannot read. */ -export function readWaits(since: number, env: NodeJS.ProcessEnv = process.env): LoggedWait[] { - const dir = waitsDirectory(env); +/** + * The lines in `dir`'s files from `since`'s UTC day on, as `parse` reads each, failing on a line it cannot read + * except a last one still being written. + */ +export function readLog(dir: string, since: number, parse: (line: string) => T | null, what: string): T[] { let names: string[]; try { names = readdirSync(dir); @@ -89,23 +88,40 @@ export function readWaits(since: number, env: NodeJS.ProcessEnv = process.env): throw error; } const first = day(since); - const waits: LoggedWait[] = []; + const read: T[] = []; for (const name of names.sort()) { if (!/^\d{4}-\d\d-\d\d\.jsonl$/.test(name) || name.slice(0, 10) < first) continue; const path = join(dir, name); const lines = readFileSync(path, "utf8").split("\n"); lines.forEach((line, i) => { if (line === "") return; - const wait = parseLogged(line); - if (wait === null) { + const value = parse(line); + if (value === null) { // A writer appends whole lines, so only a last line still being written may be cut short. if (i === lines.length - 1) return; - throw new Error(`${path}:${i + 1} is not a wait quest logged`); + throw new Error(`${path}:${i + 1} is not ${what} quest logged`); } - if (Date.parse(wait.end) >= since) waits.push(wait); + read.push(value); }); } - return waits; + return read; +} + +/** Append `wait` to today's log when the environment names a session. Throws when it cannot write. */ +export function logWait( + wait: SessionWait | GateLine, + options: { env?: NodeJS.ProcessEnv; cwd?: string; now?: number } = {}, +): void { + const env = options.env ?? process.env; + const sessions = envSessions(env); + if (sessions.length === 0) return; + const logged: LoggedWait = { ...wait, sessions, directory: options.cwd ?? process.cwd() }; + appendLog(waitsDirectory(env), logged, options.now ?? Date.now(), "wait"); +} + +/** The waits logged on this host that ended at or after `since`, failing on a line it cannot read. */ +export function readWaits(since: number, env: NodeJS.ProcessEnv = process.env): LoggedWait[] { + return readLog(waitsDirectory(env), since, parseLogged, "a wait").filter((w) => Date.parse(w.end) >= since); } function parseLogged(line: string): LoggedWait | null { @@ -117,17 +133,23 @@ function parseLogged(line: string): LoggedWait | null { } const iso = (v: unknown) => typeof v === "string" && !Number.isNaN(Date.parse(v)); const sessions = value?.sessions; - if ( - !WAIT_KINDS.includes(value?.kind as SessionWait["kind"]) || - !iso(value.start) || - !iso(value.end) || - !(value.until === null || typeof value.until === "string") || - typeof value.directory !== "string" || - !Array.isArray(sessions) || - !sessions.every((s) => typeof s?.harness === "string" && typeof s?.session === "string") - ) - return null; - return value as unknown as LoggedWait; + const shared = + iso(value?.start) && + iso(value.end) && + typeof value.directory === "string" && + Array.isArray(sessions) && + sessions.every((s) => typeof s?.harness === "string" && typeof s?.session === "string"); + if (!shared) return null; + if (value.kind === "gate") + return typeof value.command === "string" && + (value.admitted === null || iso(value.admitted)) && + (value.cpu === null || (typeof value.cpu === "number" && value.cpu >= 0)) + ? (value as unknown as LoggedWait) + : null; + return WAIT_KINDS.includes(value.kind as SessionWait["kind"]) && + (value.until === null || typeof value.until === "string") + ? (value as unknown as LoggedWait) + : null; } /** The kind of a `quest wait` that ended on `event`: the merge queue's when the PR merged or left the queue. */ diff --git a/src/core/index.ts b/src/core/index.ts index d2677d93..aea91326 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -104,12 +104,16 @@ export { parseBreakage, } from "./breakage"; export { + COMMAND_LIMIT, + type GatedRun, type ModelSpend, type RecordUpload, ROLES, type Role, type SessionRecord, type SessionWait, + type ShellCall, + type ShellGroup, SPLIT_CATEGORIES, type SplitCategory, type SplitUsage, @@ -117,6 +121,7 @@ export { NOTE_LIMIT, WAIT_KINDS, type WaitKind, + commandLine, parseRecord, parseUpload, } from "./record"; diff --git a/src/core/record.ts b/src/core/record.ts index e0dc103d..fe7d2d0a 100644 --- a/src/core/record.ts +++ b/src/core/record.ts @@ -34,6 +34,49 @@ export interface SessionWait { until: string | null; } +/** The longest command line a record keeps, in characters: enough to tell what ran, short of a heredoc's body. */ +export const COMMAND_LIMIT = 200; + +/** `text`'s first line, cut to `COMMAND_LIMIT` characters, as a record keeps a command. */ +export function commandLine(text: string): string { + const line = text.trimStart().split("\n", 1)[0]!.trimEnd(); + return line.length > COMMAND_LIMIT ? line.slice(0, COMMAND_LIMIT) : line; +} + +/** One outermost `quest gate` run, its instants ISO 8601 UTC. */ +export interface GatedRun { + /** Its command line, as `commandLine` keeps it. */ + command: string; + /** When it asked to be admitted. */ + start: string; + /** When it was admitted; null for a run that ended while queued. */ + admitted: string | null; + end: string; + /** The CPU milliseconds its command used; null where the gate ran it outside a systemd scope. */ + cpu: number | null; +} + +/** One shell call a session's harness ran, its instants ISO 8601 UTC. */ +export interface ShellCall { + /** Its command line, as `commandLine` keeps it. */ + command: string; + start: string; + /** When it returned, or for one the harness ran in the background, when it was first seen finished. */ + end: string; +} + +/** + * Shell calls that overlapped in one cgroup, and the CPU milliseconds it used from the first one's start to the + * last one's end: what ran outside `quest gate`, and the harness itself meanwhile. A group shared by sessions in + * one cgroup, as a sub-agent's with its parent's, is in each of their records. + */ +export interface ShellGroup { + /** Its calls, oldest first. */ + calls: ShellCall[]; + /** Null where the cgroup held more than the session, as a terminal's may, or none counts it, as on macOS. */ + cpu: number | null; +} + /** One model and effort's tokens in a session, and their cost at API list price in USD. */ export interface ModelSpend { model: string | null; @@ -133,6 +176,10 @@ export interface SessionRecord { end: string; /** Its timed waits, oldest first; null when they could not be read, as for a session on another host. */ waits: SessionWait[] | null; + /** Its outermost `quest gate` runs, oldest first; null for a session on another host. */ + gated: GatedRun[] | null; + /** Its other shell calls, grouped where they overlapped, oldest first; null for a session on another host. */ + shell: ShellGroup[] | null; /** Where the rest of its time went; null until every step is classified, and for a session on another host. */ split: TimeSplit | null; } @@ -175,12 +222,16 @@ export function parseRecord(value: unknown): SessionRecord { "start", "end", "waits", + "gated", + "shell", "split", ]; exact(value, keys, "the record"); const pulls = Array.isArray(value.pulls) ? value.pulls : fail("pulls"); const models = Array.isArray(value.models) ? value.models : fail("models"); const waits = value.waits === null ? null : Array.isArray(value.waits) ? value.waits : fail("waits"); + const gated = value.gated === null ? null : Array.isArray(value.gated) ? value.gated : fail("gated"); + const shell = value.shell === null ? null : Array.isArray(value.shell) ? value.shell : fail("shell"); const repository = ownerRepo(value.repository, "repository"); const quest = maybe(value.quest, "quest"); if (quest !== null && !/^quest\/.+\.md$/.test(quest)) fail("quest is not a quest path"); @@ -226,6 +277,8 @@ export function parseRecord(value: unknown): SessionRecord { fail(`a ${wait.kind} wait names what ended it`); return wait; }) ?? null, + gated: gated?.map(parseGated) ?? null, + shell: shell?.map(parseGroup) ?? null, split: value.split === null ? null : parseSplit(value.split), }; if (new Set(record.pulls).size !== record.pulls.length) fail("a PR listed twice"); @@ -234,6 +287,54 @@ export function parseRecord(value: unknown): SessionRecord { return record; } +/** A command line no longer or wider than `commandLine` keeps. */ +function command(value: unknown, field: string): string { + const line = text(value, field); + return /[\r\n]/.test(line) || line.length > COMMAND_LIMIT ? fail(`${field} is not one short line`) : line; +} + +/** CPU milliseconds, or null. */ +const cpu = (value: unknown, field: string): number | null => (value === null ? null : count(value, field)); + +/** A gated run, failing on an admission outside its run. */ +function parseGated(value: unknown): GatedRun { + if (!isObject(value)) return fail("a gated run"); + exact(value, ["command", "start", "admitted", "end", "cpu"], "a gated run"); + const run: GatedRun = { + command: command(value.command, "a gated command"), + start: instant(value.start, "a gated run's start"), + admitted: value.admitted === null ? null : instant(value.admitted, "a gated run's admission"), + end: instant(value.end, "a gated run's end"), + cpu: cpu(value.cpu, "a gated run's CPU"), + }; + const [start, end] = [Date.parse(run.start), Date.parse(run.end)]; + const admitted = run.admitted === null ? start : Date.parse(run.admitted); + if (admitted < start || end < admitted) fail("a gated run's instants are out of order"); + if (run.admitted === null && run.cpu !== null) fail("a gated run that never started used CPU"); + return run; +} + +/** A shell group, failing on one without calls or a call that ends before it starts. */ +function parseGroup(value: unknown): ShellGroup { + if (!isObject(value)) return fail("a shell group"); + exact(value, ["calls", "cpu"], "a shell group"); + if (!Array.isArray(value.calls) || value.calls.length === 0) return fail("a shell group's calls"); + return { + calls: value.calls.map((c: unknown) => { + if (!isObject(c)) return fail("a shell call"); + exact(c, ["command", "start", "end"], "a shell call"); + const call: ShellCall = { + command: command(c.command, "a shell command"), + start: instant(c.start, "a shell call's start"), + end: instant(c.end, "a shell call's end"), + }; + if (Date.parse(call.end) < Date.parse(call.start)) fail("a shell call ends before it starts"); + return call; + }), + cpu: cpu(value.cpu, "a shell group's CPU"), + }; +} + /** A split, failing on a missing or unknown category, a disagreement past what was checked, or a multi-line note. */ function parseSplit(value: unknown): TimeSplit { if (!isObject(value)) return fail("the split"); diff --git a/test/calls.test.ts b/test/calls.test.ts new file mode 100644 index 00000000..1664dfaf --- /dev/null +++ b/test/calls.test.ts @@ -0,0 +1,173 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, test } from "vitest"; + +import { type CallLine, type Sample, logCall, readCalls, sessionSample, shellGroups } from "../src/cli/calls"; +import { tempDir } from "./fixture"; + +describe.skipIf(process.platform !== "linux")("sessionSample", () => { + const scope = "/app.slice/agent.scope"; + /** A fake /proc and cgroup tree: each process's parent, and which are in the session's scope. */ + function host(parents: Record, scoped: number[], self: number) { + const proc = tempDir(); + const cgroups = tempDir(); + for (const [pid, parent] of Object.entries(parents)) { + mkdirSync(join(proc, pid)); + // The command name may hold spaces and parentheses. + writeFileSync(join(proc, pid, "stat"), `${pid} (a (b) c) S ${parent} 1 1 0\n`); + writeFileSync(join(proc, pid, "cgroup"), `0::${scoped.includes(Number(pid)) ? scope : "/other.scope"}\n`); + } + mkdirSync(join(cgroups, scope), { recursive: true }); + writeFileSync(join(cgroups, scope, "cgroup.procs"), `${scoped.join("\n")}\n`); + writeFileSync(join(cgroups, scope, "cpu.stat"), "usage_usec 7000\nuser_usec 5000\n"); + return { proc, cgroups, self }; + } + const at = Date.parse("2026-11-15T10:00:00Z"); + const counted: Sample = { at: "2026-11-15T10:00:00.000Z", cgroup: scope, usage: 7000 }; + const none: Sample = { ...counted, cgroup: null, usage: null }; + + test("counts a scope whose top is the harness, every other branch hanging from it", () => { + // The harness 10, its hook shell 20 and hook 30, an MCP server 11, a tool shell 12 and its child 13, and a + // daemon 40 whose parent left. + const parents = { 10: 1, 20: 10, 30: 20, 11: 10, 12: 10, 13: 12, 40: 1 }; + const each = host(parents, [10, 20, 30, 11, 12, 13, 40], 30); + expect(sessionSample(at, each)).toEqual(counted); + expect(sessionSample(at, { ...each, harness: 10 })).toEqual(counted); + // A harness that names itself elsewhere is not this scope's top. + expect(sessionSample(at, { ...each, harness: 20 })).toEqual(none); + }); + + test("refuses a terminal's scope, where the harness hangs below the terminal's shell", () => { + const parents = { 5: 1, 10: 5, 20: 10, 30: 20, 11: 10 }; + expect(sessionSample(at, host(parents, [5, 10, 20, 30, 11], 30))).toEqual(none); + // Another tab's work beside the session in one terminal's scope. + const shared = { 5: 1, 10: 5, 20: 10, 30: 20, 6: 5 }; + expect(sessionSample(at, { ...host(shared, [5, 10, 20, 30, 6], 30), harness: 10 })).toEqual(none); + }); + + test("refuses a hook outside the cgroup it reads", () => { + const each = host({ 10: 1, 30: 10 }, [10], 30); + expect(sessionSample(at, each)).toEqual(none); + }); +}); + +describe("shellGroups", () => { + const t = (s: number) => new Date(Date.parse("2026-11-15T10:00:00Z") + s * 1000).toISOString(); + const cg = "/agent.scope"; + const start = (call: string, s: number, usage: number | null, extra: Partial = {}): CallLine => + ({ + kind: "start", + harness: "claude", + session: "s", + agent: null, + call, + command: `run ${call}`, + at: t(s), + cgroup: cg, + usage, + ...extra, + }) as CallLine; + const end = (call: string, s: number, usage: number | null, extra: Partial = {}): CallLine => + ({ + kind: "end", + harness: "claude", + session: "s", + agent: null, + call, + task: null, + at: t(s), + cgroup: cg, + usage, + ...extra, + }) as CallLine; + const stop = (s: number, running: string[], usage: number): CallLine => ({ + kind: "stop", + harness: "claude", + session: "s", + running, + at: t(s), + cgroup: cg, + usage, + }); + + test("charges overlapping calls together, and each lone call its own span", () => { + const groups = shellGroups([ + start("a", 0, 1_000_000), + start("b", 1, 1_100_000, { agent: "sub" } as Partial), + end("a", 2, 1_500_000), + end("b", 3, 2_000_000, { agent: "sub" } as Partial), + start("c", 10, 3_000_000), + end("c", 11, 3_250_400), + // Denied or interrupted: it never returned. + start("d", 12, 3_300_000), + ]); + expect(groups.map((g) => g.group)).toEqual([ + { + calls: [ + { command: "run a", start: t(0), end: t(2) }, + { command: "run b", start: t(1), end: t(3) }, + ], + cpu: 1000, + }, + { calls: [{ command: "run c", start: t(10), end: t(11) }], cpu: 250 }, + ]); + expect(groups.map((g) => [...g.sessions])).toEqual([["claude\0s", "claude\0sub"], ["claude\0s"]]); + expect(groups[1]!.end).toBe(Date.parse(t(11))); + }); + + test("a background call runs until a turn end no longer lists its task", () => { + const lines = [ + start("bg", 0, 0), + end("bg", 1, 10_000, { task: "b1" } as Partial), + stop(5, ["b1"], 50_000), + start("fg", 6, 60_000), + end("fg", 7, 70_000), + stop(9, [], 90_000), + ]; + expect(shellGroups(lines).map((g) => [g.group.calls.map((c) => c.end), g.group.cpu])).toEqual([ + [[t(9), t(7)], 90], + ]); + // Still running at the last sample: it runs until then. + expect(shellGroups(lines.slice(0, 3)).map((g) => [g.group.calls[0]!.end, g.group.cpu])).toEqual([[t(5), 50]]); + }); + + test("a call outside a counted cgroup, or across a counter that changed, has no CPU", () => { + const uncounted = { cgroup: null, usage: null } as Partial; + const groups = shellGroups([ + start("a", 0, null, uncounted), + start("b", 0, null, uncounted), + end("a", 2, null, uncounted), + end("b", 2, null, uncounted), + start("c", 3, 5_000), + end("c", 4, null, uncounted), + start("d", 5, 9_000), + end("d", 6, 1_000), + ]); + expect(groups.map((g) => [g.group.calls.length, g.group.cpu])).toEqual([ + [1, null], + [1, null], + [1, null], + [1, null], + ]); + }); +}); + +test("the call log keeps whole lines and reads from the day before a window", () => { + const env = { XDG_STATE_HOME: tempDir() }; + const line: CallLine = { + kind: "stop", + harness: "claude", + session: "s", + running: [], + at: "2026-11-14T23:59:00.000Z", + cgroup: null, + usage: null, + }; + logCall(line, env, Date.parse(line.at)); + expect(readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toEqual([line]); + expect(readCalls(Date.parse("2026-11-16T00:30:00Z"), env)).toEqual([]); + logCall({ ...line, running: [""] }, env, Date.parse(line.at)); + logCall(line, env, Date.parse(line.at)); + expect(() => readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toThrow("is not a shell call quest logged"); +}); diff --git a/test/gate.test.ts b/test/gate.test.ts index 29db95a6..4eda8c66 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -45,6 +45,7 @@ import { import { forwards, foregroundStat, statProcess } from "../src/cli/interrupt"; import { gateBoundary, main } from "../src/cli/main"; import { alive as ownerAlive, questScope } from "../src/cli/supervise"; +import { readWaits } from "../src/cli/waits"; import { created, onTerminal, run, signalled, tempDir } from "./fixture"; // Linux reports a /proc read that races a process's exit as ESRCH rather than ENOENT, so fail @@ -1117,18 +1118,30 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { await until(() => stopped(pid())); }); - test("reports its CPU use to the gate until it ends", async () => { + test("reports its CPU use to the gate until it ends, and in all once it ends", async () => { const host = fileHost(tempDir()); const unit = `quest-gate-${randomUUID()}.scope`; const done = join(tempDir(), "done"); expect(host.usage(unit)).toBeUndefined(); const spin = ["/bin/sh", "-c", `while [ ! -e ${quote(done)} ]; do :; done`]; - const result = scope(unit)(spin, process.env, cancellation().cancel); - await until(() => (host.usage(unit) ?? 0) > 0); + const owned = scope(unit); + const result = owned(spin, process.env, cancellation().cancel); + let seen = 0; + await until(() => (seen = host.usage(unit) ?? 0) > 0); writeFileSync(done, ""); expect(await result).toBe(0); + expect(owned.cpu()).toBeGreaterThanOrEqual(Math.floor(seen / 1000)); await until(() => host.usage(unit) === undefined); }); + + test("reads a short command's CPU however soon systemd removes its scope", async () => { + // systemd removes an empty scope's cgroup as its command exits, sometimes before the gate sees the exit. + for (let i = 0; i < 20; i++) { + const owned = scope(`quest-gate-${randomUUID()}.scope`); + expect(await owned(["/bin/sh", "-c", "exit 4"], process.env, cancellation().cancel)).toBe(4); + expect(owned.cpu()).not.toBeNull(); + } + }); }); /** @@ -1581,6 +1594,31 @@ describe.skipIf(process.platform === "win32")("the run record", () => { else expect(line.unit).toBeNull(); }); + test("in an agent session, an outermost run logs its command, wait, run, and CPU for the session's record", async () => { + const state = tempDir(); + outermost(""); + vi.stubEnv("CLAUDE_CODE_SESSION_ID", "s1"); + vi.stubEnv("XDG_STATE_HOME", state); + const scoped = owner(process.env).unit !== undefined; + const spin = ["/bin/sh", "-c", "i=0; while [ $i -lt 20000 ]; do i=$((i+1)); done"]; + const before = Date.now(); + expect(await gate(spin, { host: ledger() })).toBe(0); + const [line, ...rest] = readWaits(before, { XDG_STATE_HOME: state }); + expect(rest).toEqual([]); + expect(line).toMatchObject({ + kind: "gate", + command: `/bin/sh -c ${spin[2]}`, + sessions: [{ harness: "claude", session: "s1" }], + directory: process.cwd(), + }); + if (line?.kind !== "gate") throw new Error("expected a gate line"); + const times = [before, Date.parse(line.start), Date.parse(line.admitted!), Date.parse(line.end)]; + expect(times).toEqual([...times].sort((a, b) => a - b)); + // Only a systemd scope counts its command's CPU. + if (scoped) expect(line.cpu).toBeGreaterThan(0); + else expect(line.cpu).toBeNull(); + }); + test("a nested gate records nothing, and an empty variable is unset", async () => { const dir = tempDir(); const file = join(dir, "record.jsonl"); diff --git a/test/hooks.test.ts b/test/hooks.test.ts index 6be27f2d..05ced4ed 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -23,6 +23,7 @@ import { } from "../src/cli/hooks"; import { init } from "../src/cli/setup"; import { CONTEXT } from "../src/cli/skills"; +import { readCalls } from "../src/cli/calls"; import { tempDir } from "./fixture"; const ours = { type: "command", command: HOOK_COMMAND }; @@ -220,15 +221,78 @@ test("SessionStart adds nothing where the repository does not use Quest", () => expect(hook({ hook_event_name: "SessionStart", cwd: tempDir() })).toBe(""); }); +/** Hook dependencies writing to a fresh state directory, sampling a fixed counter. */ +function sampled(usage = 5_000) { + const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir() }; + const sample = () => ({ at: "2026-11-15T10:00:00.000Z", cgroup: "/app.slice/agent.scope", usage }); + return { env, sample, lines: () => readCalls(Date.parse("2026-11-15T00:00:00Z"), env) }; +} + test("PreToolUse denies a call that breaks a checked rule, only where the repository uses Quest", () => { const cwd = checkout(`${REFERENCE_LINE}\n`); - const call = { hook_event_name: "PreToolUse", cwd, tool_name: "Bash", tool_input: { command: "git push" } }; - const output = JSON.parse(hook(call)); + const deps = sampled(); + const call = { + hook_event_name: "PreToolUse", + cwd, + session_id: "s1", + tool_use_id: "t1", + tool_name: "Bash", + tool_input: { command: "git push" }, + }; + const output = JSON.parse(hook(call, deps)); expect(output.hookSpecificOutput).toMatchObject({ hookEventName: "PreToolUse", permissionDecision: "deny" }); expect(output.hookSpecificOutput.permissionDecisionReason).toContain("refspec"); - expect(hook({ ...call, tool_input: { command: "git push origin refs/heads/a:refs/heads/a" } })).toBe(""); - expect(hook({ ...call, cwd: checkout(null) })).toBe(""); - expect(() => hook({ hook_event_name: "PreToolUse", cwd })).toThrow("tool_name"); + // A denied call never runs, so nothing samples it. + expect(deps.lines()).toEqual([]); + expect(hook({ ...call, tool_input: { command: "git push origin refs/heads/a:refs/heads/a" } }, deps)).toBe(""); + expect(hook({ ...call, cwd: checkout(null) }, deps)).toBe(""); + expect(deps.lines().map((l) => l.kind)).toEqual(["start"]); + expect(() => hook({ hook_event_name: "PreToolUse", cwd }, deps)).toThrow("tool_name"); +}); + +test("the tool hooks sample each shell call's start and return, naming its session, agent, and background task", () => { + const cwd = checkout(`${REFERENCE_LINE}\n`); + const deps = sampled(); + const claude = { cwd, session_id: "s1", agent_id: "a1", tool_use_id: "t1", tool_name: "Bash" }; + const command = `cat > x <<'EOF'\n${"y".repeat(300)}\nEOF`; + expect(hook({ ...claude, hook_event_name: "PreToolUse", tool_input: { command } }, deps)).toBe(""); + expect( + hook( + { + ...claude, + hook_event_name: "PostToolUseFailure", + tool_input: { command }, + tool_response: { backgroundTaskId: "b7" }, + }, + deps, + ), + ).toBe(""); + const codex = { cwd, session_id: "c1", turn_id: "u1", tool_use_id: "t2", tool_name: "Bash" }; + hook({ ...codex, hook_event_name: "PreToolUse", tool_input: { command: "cargo test" } }, deps); + hook( + { ...codex, hook_event_name: "PostToolUse", tool_input: { command: "cargo test" }, tool_response: "ok" }, + deps, + ); + // Edits are no shell calls. + hook({ ...claude, hook_event_name: "PostToolUse", tool_name: "Write", tool_input: {} }, deps); + const sample = { at: "2026-11-15T10:00:00.000Z", cgroup: "/app.slice/agent.scope", usage: 5_000 }; + expect(deps.lines()).toEqual([ + { + kind: "start", + harness: "claude", + session: "s1", + agent: "a1", + call: "t1", + command: "cat > x <<'EOF'", + ...sample, + }, + { kind: "end", harness: "claude", session: "s1", agent: "a1", call: "t1", task: "b7", ...sample }, + { kind: "start", harness: "codex", session: "c1", agent: null, call: "t2", command: "cargo test", ...sample }, + { kind: "end", harness: "codex", session: "c1", agent: null, call: "t2", task: null, ...sample }, + ]); + expect(() => + hook({ ...claude, tool_use_id: undefined, hook_event_name: "PostToolUse", tool_input: {} }, deps), + ).toThrow("tool_use_id"); }); /** A background task as Claude Code lists it in a Stop hook's input. */ @@ -260,9 +324,16 @@ test.each([ ], }; // A user's own state, so a sender's failure on this host is never said here; this checkout has no origin. - const deps = { env: { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir() } }; + const deps = sampled(); const output = JSON.parse(hook(stop, deps)); expect(output.decision).toBe("block"); + // Every turn end records the shell tasks still running, so a background call's end is known. + expect(deps.lines()[0]).toMatchObject({ + kind: "stop", + harness: "claude", + session: "s1", + running: ["b1", "b2", "b3"], + }); expect(output.reason).toContain("- cd repo && quest gate just test"); expect(output.reason).not.toContain("just check"); // Without a transcript, each running gate counts as the agent's. diff --git a/test/records.test.ts b/test/records.test.ts index 26cfd042..32716b66 100644 --- a/test/records.test.ts +++ b/test/records.test.ts @@ -3,7 +3,14 @@ import { join } from "node:path"; import { describe, expect, test } from "vitest"; -import { type BreakageRecord, type SessionRecord, parseBreakage, parseRecord, parseUpload } from "../src/core"; +import { + type BreakageRecord, + type SessionRecord, + type SessionWait, + parseBreakage, + parseRecord, + parseUpload, +} from "../src/core"; import { buildRecords } from "../src/cli/records"; import { transcriptFacts } from "../src/cli/transcript"; import type { Pull, Session } from "../src/cli/usage"; @@ -38,6 +45,21 @@ const record = (extra: Partial = {}): SessionRecord => ({ waits: [ { kind: "pull", start: "2026-11-06T10:10:00.000Z", end: "2026-11-06T10:20:00.000Z", until: "checks:passed" }, ], + gated: [ + { + command: "just check", + start: "2026-11-06T10:21:00.000Z", + admitted: "2026-11-06T10:22:00.000Z", + end: "2026-11-06T10:30:00.000Z", + cpu: 1200, + }, + ], + shell: [ + { + calls: [{ command: "git status", start: "2026-11-06T10:31:00.000Z", end: "2026-11-06T10:31:01.000Z" }], + cpu: 40, + }, + ], split: null, ...extra, }); @@ -92,6 +114,79 @@ describe("parseRecord", () => { "ends before it starts", ], ["a mixed-case repository", record({ repository: "Acme/Proj" }), "lowercase"], + [ + "a gated run admitted after it ended", + record({ + gated: [ + { + ...{ + command: "just check", + start: "2026-11-06T10:21:00.000Z", + admitted: "2026-11-06T10:22:00.000Z", + end: "2026-11-06T10:30:00.000Z", + cpu: 1200, + }, + admitted: "2026-11-06T10:31:00.000Z", + }, + ], + }), + "out of order", + ], + [ + "a gated run that never started but used CPU", + record({ + gated: [ + { + ...{ + command: "just check", + start: "2026-11-06T10:21:00.000Z", + admitted: "2026-11-06T10:22:00.000Z", + end: "2026-11-06T10:30:00.000Z", + cpu: 1200, + }, + admitted: null, + }, + ], + }), + "never started", + ], + [ + "a gated command over two lines", + record({ + gated: [ + { + ...{ + command: "just check", + start: "2026-11-06T10:21:00.000Z", + admitted: "2026-11-06T10:22:00.000Z", + end: "2026-11-06T10:30:00.000Z", + cpu: 1200, + }, + command: "a\nb", + }, + ], + }), + "one short line", + ], + ["a shell group without calls", record({ shell: [{ calls: [], cpu: 1 }] }), "calls"], + [ + "a shell command longer than a record keeps", + record({ + shell: [ + { + calls: [ + { + command: "x".repeat(201), + start: "2026-11-06T10:31:00.000Z", + end: "2026-11-06T10:31:01.000Z", + }, + ], + cpu: null, + }, + ], + }), + "one short line", + ], ["a quest that is not a path", record({ quest: "a.md" }), "quest path"], ])("refuses %s", (_, value, message) => { expect(() => parseRecord(value)).toThrow(message); @@ -214,6 +309,27 @@ describe("wait log", () => { expect(() => readWaits(0, e)).toThrow("2026-11-07.jsonl:2"); }); + test("logs a gate run as one line, which still reads beside the waits older gates logged", () => { + const e = env({ CLAUDE_CODE_SESSION_ID: "c1" }); + const run = { + kind: "gate" as const, + command: "just check", + start: "2026-11-06T10:00:00.000Z", + admitted: null, + end: "2026-11-06T10:05:00.000Z", + cpu: null, + }; + logWait(run, { env: e, cwd: "/w", now: Date.parse(run.end) }); + logWait(wait, { env: e, cwd: "/w", now: Date.parse(run.end) }); + expect(readWaits(0, e).map((w) => w.kind)).toEqual(["gate", "gate-run"]); + appendFileSync( + join(waitsDirectory(e), "2026-11-06.jsonl"), + `${JSON.stringify({ ...run, cpu: -1, sessions: [], directory: "/w" })}\n`, + ); + logWait(wait, { env: e, cwd: "/w", now: Date.parse(run.end) }); + expect(() => readWaits(0, e)).toThrow("2026-11-06.jsonl:3"); + }); + test("drops days past what it keeps", () => { const e = env({ CODEX_THREAD_ID: "x1" }); const dir = waitsDirectory(e); @@ -368,7 +484,7 @@ describe("buildRecords", () => { closed: null, body: "", }); - const logged = (kind: LoggedWait["kind"], start: string, sessions: string[], directory = "/w"): LoggedWait => ({ + const logged = (kind: SessionWait["kind"], start: string, sessions: string[], directory = "/w"): LoggedWait => ({ kind, start: `2026-11-06T${start}.000Z`, end: `2026-11-06T${start.replace(/:\d\d$/, ":59")}.000Z`, @@ -388,6 +504,7 @@ describe("buildRecords", () => { const p1 = pull(1, "quest/m0/a", "10:30:00"); const p2 = pull(2, "quest/m0/b", "10:05:00"); const records = buildRecords({ + calls: [], repository: "acme/proj", sessions: [coordinator, a, b, helper, loose], own: new Map([ @@ -438,6 +555,7 @@ describe("buildRecords", () => { const remote = session("far", { host: "box" }); const p = pull(3, "quest/m0/c", "09:00:00"); const records = buildRecords({ + calls: [], repository: "acme/proj", sessions: [merger, reviewer, remote], own: new Map([ @@ -471,6 +589,7 @@ describe("buildRecords", () => { const here = session("h", { parent: "p" }); const elsewhere = session("e", { parent: "p" }); const records = buildRecords({ + calls: [], repository: "acme/proj", sessions: [parent, here, elsewhere], own: new Map([[parent, pull(1, "quest/m0/a", "09:00:00")]]), @@ -488,6 +607,7 @@ describe("buildRecords", () => { const on = session("on"); const off = session("off"); const records = buildRecords({ + calls: [], repository: "acme/proj", sessions: [on, off], own: new Map([[on, pull(1, "quest/m0/a", "09:00:00")]]), @@ -504,6 +624,7 @@ describe("buildRecords", () => { // Its first request in the window comes after the wait ended. const named = session("n", { start: at("10:45:00") }); const records = buildRecords({ + calls: [], repository: "acme/proj", sessions: [anonymous, named], own: new Map([ @@ -520,6 +641,78 @@ describe("buildRecords", () => { expect(records.get(named)!.waits!.map((w) => w.kind)).toEqual(["pull"]); }); + test("a gate run gives its queue and run waits and a gated entry, and overlapping calls one shell group", () => { + const main = session("s"); + const sub = session("sub", { parent: "s", start: at("10:05:00") }); + const remote = session("r", { host: "box" }); + const gate = (start: string, admitted: string | null, end: string, cpu: number | null): LoggedWait => ({ + kind: "gate", + command: "just check", + start: `2026-11-06T${start}.000Z`, + admitted: admitted && `2026-11-06T${admitted}.000Z`, + end: `2026-11-06T${end}.000Z`, + cpu, + sessions: [{ harness: "claude", session: "s" }], + directory: "/elsewhere", + }); + const sample = (time: string, usage: number) => ({ at: `2026-11-06T${time}.000Z`, cgroup: "/a.scope", usage }); + const caller = (agent: string | null) => ({ harness: "claude", session: "s", agent }); + const records = buildRecords({ + calls: [ + { kind: "start", ...caller(null), call: "1", command: "cargo build", ...sample("10:10:00", 1_000_000) }, + { kind: "start", ...caller("sub"), call: "2", command: "ls", ...sample("10:10:01", 1_000_500) }, + { kind: "end", ...caller("sub"), call: "2", task: null, ...sample("10:10:02", 1_200_000) }, + { kind: "end", ...caller(null), call: "1", task: null, ...sample("10:10:30", 9_000_000) }, + // Before the window, so no record holds it. + { kind: "start", ...caller(null), call: "0", command: "old", ...sample("09:00:00", 0) }, + { kind: "end", ...caller(null), call: "0", task: null, ...sample("09:00:01", 10) }, + ], + repository: "acme/proj", + sessions: [main, sub, remote], + own: new Map([ + [main, pull(1, "quest/m0/a", "09:00:00")], + [remote, pull(1, "quest/m0/a", "09:00:00")], + ]), + reviewers: new Set(), + classifiers: new Set(), + waits: [gate("10:20:00", "10:21:00", "10:30:00", 1500), gate("10:40:00", null, "10:41:00", null)], + since: at("10:00:00"), + }); + const of = (s: Session) => records.get(s)!; + expect(of(main).waits!.map((w) => [w.kind, w.start.slice(11, 16), w.end.slice(11, 16)])).toEqual([ + ["gate-queue", "10:20", "10:21"], + ["gate-run", "10:21", "10:30"], + ["gate-queue", "10:40", "10:41"], + ]); + expect(of(main).gated).toEqual([ + { + command: "just check", + start: "2026-11-06T10:20:00.000Z", + admitted: "2026-11-06T10:21:00.000Z", + end: "2026-11-06T10:30:00.000Z", + cpu: 1500, + }, + { + command: "just check", + start: "2026-11-06T10:40:00.000Z", + admitted: null, + end: "2026-11-06T10:41:00.000Z", + cpu: null, + }, + ]); + const group = { + calls: [ + { command: "cargo build", start: "2026-11-06T10:10:00.000Z", end: "2026-11-06T10:10:30.000Z" }, + { command: "ls", start: "2026-11-06T10:10:01.000Z", end: "2026-11-06T10:10:02.000Z" }, + ], + cpu: 8000, + }; + expect(of(main).shell).toEqual([group]); + expect(of(sub)).toMatchObject({ gated: [], shell: [group] }); + expect(of(remote)).toMatchObject({ gated: null, shell: null }); + expect(() => parseRecord(JSON.parse(JSON.stringify(of(main))))).not.toThrow(); + }); + test("a person's wait is windowed, and an unreadable transcript leaves waits unknown", () => { const transcript = join(tempDir(), "t.jsonl"); const ask = (id: string, asked: string, answered: string) => @@ -549,6 +742,7 @@ describe("buildRecords", () => { reviewers: new Set(), classifiers: new Set(), waits: [], + calls: [], since: at("09:00:00"), }; const records = buildRecords(input); diff --git a/test/send.test.ts b/test/send.test.ts index 67dc911b..c420ddab 100644 --- a/test/send.test.ts +++ b/test/send.test.ts @@ -162,6 +162,7 @@ function sender( return sessions; }, waits: () => [], + calls: () => [], ask: async () => { throw new Error("no model can label here"); }, diff --git a/test/split.test.ts b/test/split.test.ts index a519d1b9..e2c99f92 100644 --- a/test/split.test.ts +++ b/test/split.test.ts @@ -182,6 +182,8 @@ const base = (): SessionRecord => ({ start: at(0), end: at(1), waits: [], + gated: [], + shell: [], split: { categories: Object.fromEntries(SPLIT_CATEGORIES.map((c) => [c, { ms: 1, tokens: 2 }])) as never, notes: ["one line"], diff --git a/test/usage.test.ts b/test/usage.test.ts index 27ba6b6a..0c66b0ec 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -813,6 +813,7 @@ describe("usage", () => { return exported; }, waits: () => waits, + calls: () => [], split: () => null, }); return { root, deps, sessions, calls, requests }; From 90c6fb77d78086bf52f898e22e1b2238e354f4fa Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:27:25 -0800 Subject: [PATCH 03/13] fix(records): keep each session's own calls in shared groups, and name the harness A shared shell group now lists only the recording session's calls, with a shared flag, so a sub-agent routed elsewhere never sends its commands inside its parent's record. A sample counts only when the cgroup's top is the harness Claude Code names in CLAUDE_PID, or a process named codex, so a terminal-run Codex session no longer counts its terminal's other jobs. Shell groups count from their start, which the window's logs always hold. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++++---- quest/m0/mistake-audit.md | 3 +- src/cli/calls.ts | 65 +++++++++++++++++++------------ src/cli/hooks.ts | 18 +++++---- src/cli/records.ts | 6 +-- src/core/record.ts | 14 +++++-- test/calls.test.ts | 82 ++++++++++++++++++++++++--------------- test/records.test.ts | 29 +++++++++----- 8 files changed, 145 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 18c720ea..e6c20541 100644 --- a/README.md +++ b/README.md @@ -1724,7 +1724,7 @@ latest upload, since one sender sends a session's uploads in order. The type is `parseUpload`, which refuse a malformed record whole: ```json -{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null},{"kind":"gate-run","start":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","until":null}],"gated":[{"command":"just test","start":"2026-11-14T00:22:07.386Z","admitted":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","cpu":1214300}],"shell":[{"calls":[{"command":"git status --short","start":"2026-11-14T00:21:58.004Z","end":"2026-11-14T00:21:58.051Z"}],"cpu":30}],"split":null} +{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null},{"kind":"gate-run","start":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","until":null}],"gated":[{"command":"just test","start":"2026-11-14T00:22:07.386Z","admitted":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","cpu":1214300}],"shell":[{"calls":[{"command":"git status --short","start":"2026-11-14T00:21:58.004Z","end":"2026-11-14T00:21:58.051Z"}],"cpu":30,"shared":false}],"split":null} ``` - `pulls` are the PRs it worked on, attributed as `quest usage` attributes @@ -1772,16 +1772,18 @@ latest upload, since one sender sends a session's uploads in order. The type is elsewhere. Quest's tool hooks sample that counter around each call into `$XDG_STATE_HOME/quest/calls`, kept like the waits, and only where the cgroup holds the session alone: one tree of processes whose top is the - harness, as when an app such as T3 Code starts each agent session in a - systemd scope of its own. A terminal's scope holds the terminal's shell - above the harness, so its groups, like those on macOS, have a null `cpu`. + harness, the process Claude Code names in `CLAUDE_PID` or one named + `codex`, as when an app such as T3 Code starts each agent session in a + systemd scope of its own. A terminal's scope has the terminal's shell on + top, so its groups, like those on macOS, have a null `cpu`. A call Claude Code moved to the background runs until the first turn end that no longer lists it; a call that never returned, as one denied, is left out. Codex's calls end when the tool returns, so a command it left running counts only until then. A group shared by sessions in one cgroup, - as a sub-agent's with its parent's, is in each of their records. Both - lists are null for a session on another host, and count what ended since - the window's start. + as a sub-agent's with its parent's, is in each of their records, listing + only that session's calls, with `shared` true. Both lists are null for a + session on another host; `gated` counts the runs that ended since the + window's start, and `shell` the groups that started since then. - `split` is where the rest of its time went, as the next section describes; null until every step is labeled, and for a session on another host. diff --git a/quest/m0/mistake-audit.md b/quest/m0/mistake-audit.md index f140130a..9e5bb296 100644 --- a/quest/m0/mistake-audit.md +++ b/quest/m0/mistake-audit.md @@ -57,7 +57,8 @@ Agent's choices, for the implementer to revisit. Each run: run of a usually heavy command. Count `nix build` apart, since `nix-daemon` builds outside its gate's scope. A `shell` group's `cpu` is null outside a cgroup that holds the session alone, as in a terminal, so - say how many groups were unmeasured. Records only hold calls from hosts + say how many groups were unmeasured; a `shared` group's CPU also covers + another session's calls in its cgroup, such as a sub-agent's. Records only hold calls from hosts whose hooks `quest setup` updated since that PR. - `quest messages` returns every human turn as a `kind: "message"` entry, beside the `kind: "answer"` entries tallied below, so set aside machine-written diff --git a/src/cli/calls.ts b/src/cli/calls.ts index 4a94284d..cdd09780 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -47,12 +47,9 @@ export function startLine(caller: Caller, call: string, command: string, sample: return { kind: "start", ...caller, call, command: commandLine(command), ...sample }; } -/** - * The call lines logged on this host from the UTC day before `since`'s on, so a call that started before it still - * has its start, failing on a line it cannot read. - */ +/** The call lines logged on this host from `since`'s UTC day on, failing on a line it cannot read. */ export function readCalls(since: number, env: NodeJS.ProcessEnv = process.env): CallLine[] { - return readLog(callsDirectory(env), since - 86_400_000, parseCall, "a shell call"); + return readLog(callsDirectory(env), since, parseCall, "a shell call"); } function parseCall(line: string): CallLine | null { @@ -89,21 +86,21 @@ export interface Host { proc: string; cgroups: string; self: number; - /** The harness's process, where it names it, as Claude Code does in `CLAUDE_PID`. */ - harness?: number; + /** + * The harness's process, where it names it, as Claude Code does in `CLAUDE_PID`, else the name it runs under, as + * Codex's `codex`. + */ + harness: { pid: number } | { name: string }; } /** * The cgroup holding this process and its CPU counter now, or nulls where it holds more than the hook's session. * Such a cgroup holds one tree of processes, whose top is the harness: the hook's topmost ancestor in the cgroup, from * which every other branch hangs, as when an app starts each agent session in a scope of its own. A terminal's scope - * fails, since the harness hangs below the terminal's shell, as does one shared with other work. A process whose - * parent left, as a daemonized one's does, is the session's leftover and counts. + * fails, since its top is the terminal's shell, as does one shared with other work. A process whose parent left, as + * a daemonized one's does, is the session's leftover and counts. */ -export function sessionSample( - now: number, - host: Host = { proc: "/proc", cgroups: "/sys/fs/cgroup", self: process.pid }, -): Sample { +export function sessionSample(now: number, host: Host): Sample { const at = new Date(now).toISOString(); const none = { at, cgroup: null, usage: null }; if (process.platform !== "linux") return none; @@ -125,7 +122,7 @@ export function sessionSample( chain.add(pid); top = pid; } - if (!chain.has(host.self) || (host.harness !== undefined && host.harness !== top)) return none; + if (!chain.has(host.self) || !harnessIs(host, top)) return none; for (const pid of inside) { if (chain.has(pid)) continue; // The chain member this process hangs from, or undefined for a tree whose root's parent left the cgroup. @@ -137,6 +134,17 @@ export function sessionSample( return usage === null ? none : { at, cgroup: path, usage }; } +/** Whether `pid` is the harness `host` names, by its process ID or its name. */ +function harnessIs(host: Host, pid: number): boolean { + if ("pid" in host.harness) return host.harness.pid === pid; + try { + return readFileSync(join(host.proc, String(pid), "comm"), "utf8").trim() === host.harness.name; + } catch (error) { + if (["ENOENT", "ESRCH"].includes((error as NodeJS.ErrnoException).code ?? "")) return false; + throw error; + } +} + /** Field 4 of /proc//stat, or undefined once the process is gone. */ function parentOf(proc: string, pid: number): number | undefined { let stat: string; @@ -149,12 +157,19 @@ function parentOf(proc: string, pid: number): number | undefined { return Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[1]); } -/** A shell group, and the sessions, as `harness\0session` with a Claude sub-agent's agent ID, that ran its calls. */ +/** Calls that overlapped in one cgroup, each with its session, and what the cgroup used meanwhile. */ export interface OwnedGroup { - group: ShellGroup; - sessions: Set; - /** Its last call's end, epoch ms. */ - end: number; + /** Each call, oldest first, with its session as `harness\0session`, a Claude sub-agent's being its agent ID. */ + calls: { session: string; call: ShellCall }[]; + cpu: number | null; + /** Its first call's start, epoch ms. */ + start: number; +} + +/** `group` as `session`'s record holds it: only its own calls, and whether others ran in it too. */ +export function sessionGroup(group: OwnedGroup, session: string): ShellGroup | null { + const own = group.calls.filter((c) => c.session === session).map((c) => c.call); + return own.length ? { calls: own, cpu: group.cpu, shared: own.length < group.calls.length } : null; } type Start = Extract; @@ -216,12 +231,12 @@ export function shellGroups(lines: CallLine[]): OwnedGroup[] { end.usage !== null && end.usage >= first.usage; return { - group: { - calls: grouped.map((c): ShellCall => ({ command: c.start.command, start: c.start.at, end: c.end.at })), - cpu: counted ? Math.round((end.usage! - first.usage!) / 1000) : null, - }, - sessions: new Set(grouped.map((c) => `${c.start.harness}\0${c.start.agent ?? c.start.session}`)), - end: ms(end), + calls: grouped.map((c) => ({ + session: `${c.start.harness}\0${c.start.agent ?? c.start.session}`, + call: { command: c.start.command, start: c.start.at, end: c.end.at }, + })), + cpu: counted ? Math.round((end.usage! - first.usage!) / 1000) : null, + start: ms(first), }; }); } diff --git a/src/cli/hooks.ts b/src/cli/hooks.ts index 0d724d61..020f3e1d 100644 --- a/src/cli/hooks.ts +++ b/src/cli/hooks.ts @@ -424,13 +424,17 @@ function callId(value: Partial): string { /** The session's sample now; Claude Code names its own process, which an agent it starts inherits. */ function sample(deps: HookDependencies, value: Partial): Sample { if (deps.sample) return deps.sample(); - const harness = typeof value.turn_id === "string" ? NaN : Number(deps.env.CLAUDE_PID); - return sessionSample(Date.now(), { - proc: "/proc", - cgroups: "/sys/fs/cgroup", - self: process.pid, - ...(Number.isSafeInteger(harness) && harness > 0 ? { harness } : {}), - }); + const now = Date.now(); + const pid = Number(deps.env.CLAUDE_PID); + // Without its process ID, Claude Code runs under no name a shell could not share, so nothing is counted. + const harness = + typeof value.turn_id === "string" + ? { name: "codex" } + : Number.isSafeInteger(pid) && pid > 0 + ? { pid } + : undefined; + if (!harness) return { at: new Date(now).toISOString(), cgroup: null, usage: null }; + return sessionSample(now, { proc: "/proc", cgroups: "/sys/fs/cgroup", self: process.pid, harness }); } /** diff --git a/src/cli/records.ts b/src/cli/records.ts index 985f9952..066f013f 100644 --- a/src/cli/records.ts +++ b/src/cli/records.ts @@ -1,6 +1,6 @@ // Each session's record, as `quest usage` prints it and the stop hook sends it: its PRs, role, spend, and waits. import { type GatedRun, type Role, type SessionRecord, type SessionWait, type TimeSplit, branchQuest } from "../core"; -import { type CallLine, shellGroups } from "./calls"; +import { type CallLine, sessionGroup, shellGroups } from "./calls"; import { type TranscriptFacts, transcriptFacts } from "./transcript"; import type { Pull, Session } from "./usage"; import type { LoggedWait } from "./waits"; @@ -129,7 +129,7 @@ export function buildRecords(input: RecordInput): Map { }; const waits = assignWaits(named, input.waits, parentOf, input.since); - const groups = shellGroups(input.calls).filter((g) => g.end >= input.since); + const groups = shellGroups(input.calls).filter((g) => g.start >= input.since); const records = new Map(); for (const s of input.sessions) { const { pulls, via } = pullsOf(s); @@ -184,7 +184,7 @@ export function buildRecords(input: RecordInput): Map { gated: s.host === null ? gated : null, shell: s.host === null && s.id !== null - ? groups.filter((g) => g.sessions.has(id(s.harness, s.id!))).map((g) => g.group) + ? groups.flatMap((g) => sessionGroup(g, id(s.harness, s.id!)) ?? []) : s.host === null ? [] : null, diff --git a/src/core/record.ts b/src/core/record.ts index fe7d2d0a..9ee7f0bf 100644 --- a/src/core/record.ts +++ b/src/core/record.ts @@ -68,13 +68,15 @@ export interface ShellCall { /** * Shell calls that overlapped in one cgroup, and the CPU milliseconds it used from the first one's start to the * last one's end: what ran outside `quest gate`, and the harness itself meanwhile. A group shared by sessions in - * one cgroup, as a sub-agent's with its parent's, is in each of their records. + * one cgroup, as a sub-agent's with its parent's, is in each of their records, listing only that session's calls. */ export interface ShellGroup { - /** Its calls, oldest first. */ + /** The session's calls in it, oldest first. */ calls: ShellCall[]; /** Null where the cgroup held more than the session, as a terminal's may, or none counts it, as on macOS. */ cpu: number | null; + /** Whether calls of other sessions in its cgroup ran in it too, sharing its CPU. */ + shared: boolean; } /** One model and effort's tokens in a session, and their cost at API list price in USD. */ @@ -178,7 +180,10 @@ export interface SessionRecord { waits: SessionWait[] | null; /** Its outermost `quest gate` runs, oldest first; null for a session on another host. */ gated: GatedRun[] | null; - /** Its other shell calls, grouped where they overlapped, oldest first; null for a session on another host. */ + /** + * Its shell calls, grouped where they overlapped, from the groups that started since the window's start, oldest + * first; null for a session on another host. + */ shell: ShellGroup[] | null; /** Where the rest of its time went; null until every step is classified, and for a session on another host. */ split: TimeSplit | null; @@ -317,7 +322,7 @@ function parseGated(value: unknown): GatedRun { /** A shell group, failing on one without calls or a call that ends before it starts. */ function parseGroup(value: unknown): ShellGroup { if (!isObject(value)) return fail("a shell group"); - exact(value, ["calls", "cpu"], "a shell group"); + exact(value, ["calls", "cpu", "shared"], "a shell group"); if (!Array.isArray(value.calls) || value.calls.length === 0) return fail("a shell group's calls"); return { calls: value.calls.map((c: unknown) => { @@ -332,6 +337,7 @@ function parseGroup(value: unknown): ShellGroup { return call; }), cpu: cpu(value.cpu, "a shell group's CPU"), + shared: flag(value.shared, "whether a shell group is shared"), }; } diff --git a/test/calls.test.ts b/test/calls.test.ts index 1664dfaf..1c1a7477 100644 --- a/test/calls.test.ts +++ b/test/calls.test.ts @@ -3,12 +3,20 @@ import { join } from "node:path"; import { describe, expect, test } from "vitest"; -import { type CallLine, type Sample, logCall, readCalls, sessionSample, shellGroups } from "../src/cli/calls"; +import { + type CallLine, + type Sample, + logCall, + readCalls, + sessionGroup, + sessionSample, + shellGroups, +} from "../src/cli/calls"; import { tempDir } from "./fixture"; describe.skipIf(process.platform !== "linux")("sessionSample", () => { const scope = "/app.slice/agent.scope"; - /** A fake /proc and cgroup tree: each process's parent, and which are in the session's scope. */ + /** A fake /proc and cgroup tree: each process's parent, which are in the session's scope, and the name of 10. */ function host(parents: Record, scoped: number[], self: number) { const proc = tempDir(); const cgroups = tempDir(); @@ -16,6 +24,7 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { mkdirSync(join(proc, pid)); // The command name may hold spaces and parentheses. writeFileSync(join(proc, pid, "stat"), `${pid} (a (b) c) S ${parent} 1 1 0\n`); + writeFileSync(join(proc, pid, "comm"), pid === "10" ? "codex\n" : "bash\n"); writeFileSync(join(proc, pid, "cgroup"), `0::${scoped.includes(Number(pid)) ? scope : "/other.scope"}\n`); } mkdirSync(join(cgroups, scope), { recursive: true }); @@ -32,23 +41,27 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { // daemon 40 whose parent left. const parents = { 10: 1, 20: 10, 30: 20, 11: 10, 12: 10, 13: 12, 40: 1 }; const each = host(parents, [10, 20, 30, 11, 12, 13, 40], 30); - expect(sessionSample(at, each)).toEqual(counted); - expect(sessionSample(at, { ...each, harness: 10 })).toEqual(counted); - // A harness that names itself elsewhere is not this scope's top. - expect(sessionSample(at, { ...each, harness: 20 })).toEqual(none); + expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(counted); + expect(sessionSample(at, { ...each, harness: { name: "codex" } })).toEqual(counted); + // A harness that names itself, or runs under a name, elsewhere is not this scope's top. + expect(sessionSample(at, { ...each, harness: { pid: 20 } })).toEqual(none); + expect(sessionSample(at, { ...each, harness: { name: "claude" } })).toEqual(none); }); - test("refuses a terminal's scope, where the harness hangs below the terminal's shell", () => { - const parents = { 5: 1, 10: 5, 20: 10, 30: 20, 11: 10 }; - expect(sessionSample(at, host(parents, [5, 10, 20, 30, 11], 30))).toEqual(none); - // Another tab's work beside the session in one terminal's scope. - const shared = { 5: 1, 10: 5, 20: 10, 30: 20, 6: 5 }; - expect(sessionSample(at, { ...host(shared, [5, 10, 20, 30, 6], 30), harness: 10 })).toEqual(none); + test("refuses a terminal's scope, whose top is the terminal's shell", () => { + // The shell 5 runs the harness 10 and a background job 6, which hangs from the top as the harness's would. + const parents = { 5: 1, 10: 5, 20: 10, 30: 20, 6: 5 }; + const each = host(parents, [5, 10, 20, 30, 6], 30); + expect(sessionSample(at, { ...each, harness: { name: "codex" } })).toEqual(none); + expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); + // A branch below the hook's chain but not from its top shares the cgroup too. + const below = { 10: 1, 20: 10, 30: 20, 21: 20 }; + expect(sessionSample(at, { ...host(below, [10, 20, 30, 21], 30), harness: { pid: 10 } })).toEqual(none); }); test("refuses a hook outside the cgroup it reads", () => { const each = host({ 10: 1, 30: 10 }, [10], 30); - expect(sessionSample(at, each)).toEqual(none); + expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); }); }); @@ -102,18 +115,25 @@ describe("shellGroups", () => { // Denied or interrupted: it never returned. start("d", 12, 3_300_000), ]); - expect(groups.map((g) => g.group)).toEqual([ - { - calls: [ - { command: "run a", start: t(0), end: t(2) }, - { command: "run b", start: t(1), end: t(3) }, + expect(groups.map((g) => [g.calls.map((c) => [c.session, c.call]), g.cpu, g.start])).toEqual([ + [ + [ + ["claude\0s", { command: "run a", start: t(0), end: t(2) }], + ["claude\0sub", { command: "run b", start: t(1), end: t(3) }], ], - cpu: 1000, - }, - { calls: [{ command: "run c", start: t(10), end: t(11) }], cpu: 250 }, + 1000, + Date.parse(t(0)), + ], + [[["claude\0s", { command: "run c", start: t(10), end: t(11) }]], 250, Date.parse(t(10))], ]); - expect(groups.map((g) => [...g.sessions])).toEqual([["claude\0s", "claude\0sub"], ["claude\0s"]]); - expect(groups[1]!.end).toBe(Date.parse(t(11))); + // Each session's record holds its own calls, and says another's shared the group's CPU. + expect(sessionGroup(groups[0]!, "claude\0sub")).toEqual({ + calls: [{ command: "run b", start: t(1), end: t(3) }], + cpu: 1000, + shared: true, + }); + expect(sessionGroup(groups[1]!, "claude\0s")).toMatchObject({ shared: false }); + expect(sessionGroup(groups[1]!, "claude\0sub")).toBeNull(); }); test("a background call runs until a turn end no longer lists its task", () => { @@ -125,11 +145,9 @@ describe("shellGroups", () => { end("fg", 7, 70_000), stop(9, [], 90_000), ]; - expect(shellGroups(lines).map((g) => [g.group.calls.map((c) => c.end), g.group.cpu])).toEqual([ - [[t(9), t(7)], 90], - ]); + expect(shellGroups(lines).map((g) => [g.calls.map((c) => c.call.end), g.cpu])).toEqual([[[t(9), t(7)], 90]]); // Still running at the last sample: it runs until then. - expect(shellGroups(lines.slice(0, 3)).map((g) => [g.group.calls[0]!.end, g.group.cpu])).toEqual([[t(5), 50]]); + expect(shellGroups(lines.slice(0, 3)).map((g) => [g.calls[0]!.call.end, g.cpu])).toEqual([[t(5), 50]]); }); test("a call outside a counted cgroup, or across a counter that changed, has no CPU", () => { @@ -144,7 +162,7 @@ describe("shellGroups", () => { start("d", 5, 9_000), end("d", 6, 1_000), ]); - expect(groups.map((g) => [g.group.calls.length, g.group.cpu])).toEqual([ + expect(groups.map((g) => [g.calls.length, g.cpu])).toEqual([ [1, null], [1, null], [1, null], @@ -153,7 +171,7 @@ describe("shellGroups", () => { }); }); -test("the call log keeps whole lines and reads from the day before a window", () => { +test("the call log keeps whole lines and reads from a window's day", () => { const env = { XDG_STATE_HOME: tempDir() }; const line: CallLine = { kind: "stop", @@ -165,9 +183,9 @@ test("the call log keeps whole lines and reads from the day before a window", () usage: null, }; logCall(line, env, Date.parse(line.at)); - expect(readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toEqual([line]); - expect(readCalls(Date.parse("2026-11-16T00:30:00Z"), env)).toEqual([]); + expect(readCalls(Date.parse("2026-11-14T20:00:00Z"), env)).toEqual([line]); + expect(readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toEqual([]); logCall({ ...line, running: [""] }, env, Date.parse(line.at)); logCall(line, env, Date.parse(line.at)); - expect(() => readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toThrow("is not a shell call quest logged"); + expect(() => readCalls(Date.parse("2026-11-14T20:00:00Z"), env)).toThrow("is not a shell call quest logged"); }); diff --git a/test/records.test.ts b/test/records.test.ts index 32716b66..bdc642fd 100644 --- a/test/records.test.ts +++ b/test/records.test.ts @@ -58,6 +58,7 @@ const record = (extra: Partial = {}): SessionRecord => ({ { calls: [{ command: "git status", start: "2026-11-06T10:31:00.000Z", end: "2026-11-06T10:31:01.000Z" }], cpu: 40, + shared: false, }, ], split: null, @@ -168,7 +169,7 @@ describe("parseRecord", () => { }), "one short line", ], - ["a shell group without calls", record({ shell: [{ calls: [], cpu: 1 }] }), "calls"], + ["a shell group without calls", record({ shell: [{ calls: [], cpu: 1, shared: false }] }), "calls"], [ "a shell command longer than a record keeps", record({ @@ -182,6 +183,7 @@ describe("parseRecord", () => { }, ], cpu: null, + shared: false, }, ], }), @@ -700,15 +702,24 @@ describe("buildRecords", () => { cpu: null, }, ]); - const group = { - calls: [ - { command: "cargo build", start: "2026-11-06T10:10:00.000Z", end: "2026-11-06T10:10:30.000Z" }, - { command: "ls", start: "2026-11-06T10:10:01.000Z", end: "2026-11-06T10:10:02.000Z" }, + // Each holds its own calls, so no session's commands reach another's record, and the group's CPU. + expect(of(main).shell).toEqual([ + { + calls: [{ command: "cargo build", start: "2026-11-06T10:10:00.000Z", end: "2026-11-06T10:10:30.000Z" }], + cpu: 8000, + shared: true, + }, + ]); + expect(of(sub)).toMatchObject({ + gated: [], + shell: [ + { + calls: [{ command: "ls", start: "2026-11-06T10:10:01.000Z", end: "2026-11-06T10:10:02.000Z" }], + cpu: 8000, + shared: true, + }, ], - cpu: 8000, - }; - expect(of(main).shell).toEqual([group]); - expect(of(sub)).toMatchObject({ gated: [], shell: [group] }); + }); expect(of(remote)).toMatchObject({ gated: null, shell: null }); expect(() => parseRecord(JSON.parse(JSON.stringify(of(main))))).not.toThrow(); }); From c4ab72ce4d0f95cb9dfdda1436730cbf39049b84 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:31:37 -0800 Subject: [PATCH 04/13] fix(records): read earlier call days a window's calls began on Co-Authored-By: Claude Opus 5.5 --- src/cli/calls.ts | 36 +++++++++++++++++++++++++++++++++--- src/cli/waits.ts | 34 +++++++++++++++++++++++++--------- test/calls.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/src/cli/calls.ts b/src/cli/calls.ts index cdd09780..67767881 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -6,7 +6,7 @@ import { join } from "node:path/posix"; import { type ShellCall, type ShellGroup, commandLine } from "../core"; import { cgroupUsage, members } from "./supervise"; -import { appendLog, logDirectory, readLog } from "./waits"; +import { appendLog, logDays, logDirectory, readDays } from "./waits"; /** The session's cgroup and its CPU counter, in microseconds, when a hook sampled them. */ export interface Sample { @@ -47,9 +47,39 @@ export function startLine(caller: Caller, call: string, command: string, sample: return { kind: "start", ...caller, call, command: commandLine(command), ...sample }; } -/** The call lines logged on this host from `since`'s UTC day on, failing on a line it cannot read. */ +/** + * The call lines logged on this host from `since`'s UTC day on, and from each earlier day while one of them is about + * a call whose start or background handoff lies earlier, so a group that began before the window ends where it + * did. Fails on a line it cannot read. + */ export function readCalls(since: number, env: NodeJS.ProcessEnv = process.env): CallLine[] { - return readLog(callsDirectory(env), since, parseCall, "a shell call"); + const dir = callsDirectory(env); + const days = logDays(dir); + const first = new Date(since).toISOString().slice(0, 10); + let from = days.findIndex((name) => name >= first); + if (from === -1) from = days.length; + let lines = readDays(dir, days.slice(from), parseCall, "a shell call"); + while (from > 0 && begunEarlier(lines)) { + from--; + lines = [...readDays(dir, [days[from]!], parseCall, "a shell call"), ...lines]; + } + return lines; +} + +/** Whether `lines` hold a call's return without its start, or a running background task without its handoff. */ +function begunEarlier(lines: CallLine[]): boolean { + const started = new Set(); + const handed = new Set(); + for (const line of lines) { + if (line.kind === "start") started.add(`${line.harness}\0${line.session}\0${line.call}`); + if (line.kind === "end" && line.task !== null) handed.add(`${line.harness}\0${line.session}\0${line.task}`); + } + return lines.some((line) => + line.kind === "end" + ? !started.has(`${line.harness}\0${line.session}\0${line.call}`) + : line.kind === "stop" && + line.running.some((task) => !handed.has(`${line.harness}\0${line.session}\0${task}`)), + ); } function parseCall(line: string): CallLine | null { diff --git a/src/cli/waits.ts b/src/cli/waits.ts index d0cbd9b3..1349a75b 100644 --- a/src/cli/waits.ts +++ b/src/cli/waits.ts @@ -75,11 +75,8 @@ export function appendLog(dir: string, value: unknown, now: number, what: string } } -/** - * The lines in `dir`'s files from `since`'s UTC day on, as `parse` reads each, failing on a line it cannot read - * except a last one still being written. - */ -export function readLog(dir: string, since: number, parse: (line: string) => T | null, what: string): T[] { +/** The days `dir` holds a file for, as `YYYY-MM-DD`, oldest first. */ +export function logDays(dir: string): string[] { let names: string[]; try { names = readdirSync(dir); @@ -87,11 +84,30 @@ export function readLog(dir: string, since: number, parse: (line: string) => if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } - const first = day(since); + return names + .filter((name) => /^\d{4}-\d\d-\d\d\.jsonl$/.test(name)) + .map((name) => name.slice(0, 10)) + .sort(); +} + +/** + * The lines in `dir`'s files from `since`'s UTC day on, as `parse` reads each, failing on a line it cannot read + * except a last one still being written. + */ +export function readLog(dir: string, since: number, parse: (line: string) => T | null, what: string): T[] { + return readDays( + dir, + logDays(dir).filter((name) => name >= day(since)), + parse, + what, + ); +} + +/** The lines in `dir`'s files for `days`, as `readLog` reads them. */ +export function readDays(dir: string, days: string[], parse: (line: string) => T | null, what: string): T[] { const read: T[] = []; - for (const name of names.sort()) { - if (!/^\d{4}-\d\d-\d\d\.jsonl$/.test(name) || name.slice(0, 10) < first) continue; - const path = join(dir, name); + for (const name of days) { + const path = join(dir, `${name}.jsonl`); const lines = readFileSync(path, "utf8").split("\n"); lines.forEach((line, i) => { if (line === "") return; diff --git a/test/calls.test.ts b/test/calls.test.ts index 1c1a7477..bb8f0733 100644 --- a/test/calls.test.ts +++ b/test/calls.test.ts @@ -189,3 +189,29 @@ test("the call log keeps whole lines and reads from a window's day", () => { logCall(line, env, Date.parse(line.at)); expect(() => readCalls(Date.parse("2026-11-14T20:00:00Z"), env)).toThrow("is not a shell call quest logged"); }); + +test("reading a window also reads each earlier day a call in it began on", () => { + const env = { XDG_STATE_HOME: tempDir() }; + const sample = (at: string) => ({ at, cgroup: "/a.scope", usage: 0 }); + const caller = { harness: "claude", session: "s", agent: null }; + const log = (line: CallLine) => logCall(line, env, Date.parse(line.at)); + // Two days back, an unrelated call; the day before, a background call handed off; the window's day, a call that + // ran over midnight and a turn end that still lists the background task. + log({ kind: "start", ...caller, call: "old", command: "old", ...sample("2026-11-13T10:00:00.000Z") }); + log({ kind: "end", ...caller, call: "old", task: null, ...sample("2026-11-13T10:00:01.000Z") }); + log({ kind: "start", ...caller, call: "bg", command: "bg", ...sample("2026-11-14T23:50:00.000Z") }); + log({ kind: "end", ...caller, call: "bg", task: "b1", ...sample("2026-11-14T23:50:01.000Z") }); + log({ kind: "start", ...caller, call: "late", command: "late", ...sample("2026-11-14T23:59:00.000Z") }); + log({ kind: "end", ...caller, call: "late", task: null, ...sample("2026-11-15T00:01:00.000Z") }); + log({ kind: "stop", harness: "claude", session: "s", running: ["b1"], ...sample("2026-11-15T00:02:00.000Z") }); + const read = readCalls(Date.parse("2026-11-15T00:00:00Z"), env); + expect(read.map((l) => (l.kind === "stop" ? "stop" : `${l.kind} ${l.call}`))).toEqual([ + "start bg", + "end bg", + "start late", + "end late", + "stop", + ]); + // The group they form began before the window, so no record of the window holds it. + expect(shellGroups(read).map((g) => [g.start, g.calls.length])).toEqual([[Date.parse("2026-11-14T23:50:00Z"), 2]]); +}); From cb191023a4201b6f3d29f41930d2c6c47104bce9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:40:58 -0800 Subject: [PATCH 05/13] fix(records): keep only program names in commands, and read the day before a window A command line can carry a secret such as an inline token, and records leave the host. Records now keep each program's name, and the subcommand of those that take one, never arguments; the parser refuses anything else. `quest usage --since` now reads the call log from the day before the window's, so a background call handed off then and finished before the window's first turn end no longer splits its group. Co-Authored-By: Claude Opus 5.5 --- README.md | 25 +++-- quest/m0/mistake-audit.md | 4 +- src/cli/calls.ts | 17 +-- src/cli/gate.ts | 4 +- src/core/command.ts | 231 ++++++++++++++++++++++++++++++++++++++ src/core/index.ts | 3 +- src/core/record.ts | 20 +--- test/calls.test.ts | 22 +++- test/command.test.ts | 44 ++++++++ test/gate.test.ts | 3 +- test/hooks.test.ts | 2 +- test/records.test.ts | 12 +- 12 files changed, 344 insertions(+), 43 deletions(-) create mode 100644 src/core/command.ts create mode 100644 test/command.test.ts diff --git a/README.md b/README.md index 66a4973c..b7a92f27 100644 --- a/README.md +++ b/README.md @@ -1802,15 +1802,15 @@ latest upload, since one sender sends a session's uploads in order. The type is session on another host in `quest usage --ssh`, or one whose transcript has lines token-max could not read either. Like token costs, waits count only those that ended since the window's start. -- `gated` lists each outermost `quest gate` run: its `command`, cut to its - first line and 200 characters, when it asked to start, when it was - `admitted` (null when it ended queued), when it ended, and the CPU - milliseconds its command used, read from its systemd scope once it exits, - or null where the gate owns no scope, as on macOS or Windows. The gate logs +- `gated` lists each outermost `quest gate` run: its `command`, when it + asked to start, when it was `admitted` (null when it ended queued), when it + ended, and the CPU milliseconds its command used, read from its systemd + scope once it exits, or null where the gate owns no scope, as on macOS or + Windows. The gate logs each run as one line among the waits, and the record takes its `gate-queue` and `gate-run` waits from it. `nix build` counts little here, since `nix-daemon` builds outside the scope. -- `shell` lists its shell calls, each cut the same way, in groups: calls that +- `shell` lists its shell calls, each with its `command`, in groups: calls that overlapped, in parallel or in the background, are one group. A group's `cpu` is the milliseconds the session's own cgroup used from its first call's start to its last call's end, which counts what ran outside the @@ -1829,7 +1829,18 @@ latest upload, since one sender sends a session's uploads in order. The type is as a sub-agent's with its parent's, is in each of their records, listing only that session's calls, with `shared` true. Both lists are null for a session on another host; `gated` counts the runs that ended since the - window's start, and `shell` the groups that started since then. + window's start, and `shell` the groups that started since then. To find + where a group began, `quest usage` reads the call log from the day before + the window's, and earlier days while a call it read began on one; only a + background call handed off and finished within one turn of over a day can + still split a group at the window's start. +- A `command` keeps only the programs it ran, in order and each once, with + the subcommand of those that take one, such as `cd; cargo build; tail`: + never arguments, assignments, redirections, or here-document bodies, since + a command line can carry a secret and records leave the host. Programs + such as `git`, `cargo`, `just`, `nix`, and `npm` keep their subcommand; + any other's first argument is data and stays. A program written as a + variable is `?`. - `split` is where the rest of its time went, as the next section describes; null until every step is labeled, and for a session on another host. diff --git a/quest/m0/mistake-audit.md b/quest/m0/mistake-audit.md index 9e5bb296..5a564094 100644 --- a/quest/m0/mistake-audit.md +++ b/quest/m0/mistake-audit.md @@ -54,7 +54,9 @@ Agent's choices, for the implementer to revisit. Each run: - Read both kinds of gating mistake from each record's `gated` and `shell` lists: gated runs ranked by least CPU, and shell groups by most, with no threshold, judging each command's class rather than one cheap - run of a usually heavy command. Count `nix build` apart, since + run of a usually heavy command. A `command` holds only program names and + subcommands, such as `cd; cargo build`, so judge the class by those. Count + `nix build` apart, since `nix-daemon` builds outside its gate's scope. A `shell` group's `cpu` is null outside a cgroup that holds the session alone, as in a terminal, so say how many groups were unmeasured; a `shared` group's CPU also covers diff --git a/src/cli/calls.ts b/src/cli/calls.ts index 67767881..ec866649 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -4,7 +4,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path/posix"; -import { type ShellCall, type ShellGroup, commandLine } from "../core"; +import { type ShellCall, type ShellGroup, commandName } from "../core"; import { cgroupUsage, members } from "./supervise"; import { appendLog, logDays, logDirectory, readDays } from "./waits"; @@ -42,20 +42,23 @@ export function logCall(line: CallLine, env: NodeJS.ProcessEnv = process.env, no appendLog(callsDirectory(env), line, now, "shell call"); } -/** A start line for `command`, cut as a record keeps it. */ +/** A start line for `command`, as a record keeps it: the programs it runs, without arguments. */ export function startLine(caller: Caller, call: string, command: string, sample: Sample): CallLine { - return { kind: "start", ...caller, call, command: commandLine(command), ...sample }; + return { kind: "start", ...caller, call, command: commandName(command), ...sample }; } /** - * The call lines logged on this host from `since`'s UTC day on, and from each earlier day while one of them is about - * a call whose start or background handoff lies earlier, so a group that began before the window ends where it - * did. Fails on a line it cannot read. + * The call lines logged on this host from the UTC day before `since`'s on, and from each earlier day while one of + * them is about a call whose start or background handoff lies earlier, so a group that began before the window ends + * where it did. Fails on a line it cannot read. */ export function readCalls(since: number, env: NodeJS.ProcessEnv = process.env): CallLine[] { const dir = callsDirectory(env); const days = logDays(dir); - const first = new Date(since).toISOString().slice(0, 10); + // The day before the window's, too: a background call handed off then and finished before the window's first turn + // end leaves no line in the window naming it, yet ends the group a call in the window overlapped. Only one turn + // running over a day hides such a call still. + const first = new Date(since - 86_400_000).toISOString().slice(0, 10); let from = days.findIndex((name) => name >= first); if (from === -1) from = days.length; let lines = readDays(dir, days.slice(from), parseCall, "a shell call"); diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 942ab125..069702d9 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -24,7 +24,7 @@ import type { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; import { stubbed } from "./job"; -import { commandLine } from "../core"; +import { argvName } from "../core"; import { envSessions, logWait } from "./waits"; import { cgroupUsage, measuredRun, questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; @@ -930,7 +930,7 @@ export async function gate(command: string[], options: GateOptions = {}): Promis logWait( { kind: "gate", - command: commandLine(command.join(" ")), + command: argvName(command), start: iso(started), admitted: observed.admitted === undefined ? null : iso(observed.admitted), end: iso(Date.now()), diff --git a/src/core/command.ts b/src/core/command.ts new file mode 100644 index 00000000..71c9cad5 --- /dev/null +++ b/src/core/command.ts @@ -0,0 +1,231 @@ +// What a session record keeps of a command it ran: each program's name and, where it takes one, its subcommand, such +// as `cd; cargo build; tail`. Arguments, assignments, redirections, and quoted text stay on the host, since a +// command line can carry a secret, such as an inline token, and records leave it. That is still enough for the +// mistake audit to tell a cheap command from a heavy one: `cargo check` from `cargo build`, `just check` from `just +// test`. + +/** The longest name a record keeps, in characters; names past it are left out. */ +export const COMMAND_LIMIT = 200; + +/** A program's name: a path's last part, short and of plain characters, or `?` for one written otherwise. */ +const PROGRAM = /^(?:[A-Za-z0-9._+][A-Za-z0-9._+-]{0,31}|\?)$/; +/** A subcommand: lowercase words joined by hyphens, as `build` or `merge-base`, never a value such as a token. */ +const SUBCOMMAND = /^[a-z]{1,24}(?:-[a-z]{1,24})*$/; +const SUBCOMMAND_LIMIT = 24; + +/** + * Programs whose first argument names what they do, kept as its subcommand. Any other program's first argument is + * data, such as `cd`'s directory or `echo`'s text, and stays on the host. + */ +const SUBCOMMANDS = new Set([ + "apt", + "brew", + "bun", + "bunx", + "cargo", + "deno", + "dnf", + "docker", + "gh", + "git", + "go", + "just", + "kubectl", + "make", + "nix", + "npm", + "npx", + "pnpm", + "podman", + "quest", + "rustup", + "systemctl", + "uv", + "yarn", +]); + +/** Words that open a compound command or a wrapper of the next one, skipped to reach the program it runs. */ +const KEYWORDS = new Set(["if", "then", "else", "elif", "do", "while", "until", "!", "{", "}"]); +/** Words that run the command after them, with flags and a number, such as a priority or timeout, between. */ +const WRAPPERS = new Set(["env", "sudo", "time", "nice", "nohup", "exec", "command", "timeout", "xargs"]); +/** Words that open a construct naming no program, whose simple command is left out. */ +const SKIPPED = new Set(["for", "case", "select", "function", "done", "fi", "esac", "in", "[", "[["]); + +/** Whether `name` has the form `commandName` and `argvName` give, so a record can carry it. */ +export function isCommandName(name: string): boolean { + return ( + name.length > 0 && + name.length <= COMMAND_LIMIT && + name.split("; ").every((part) => { + const [program, sub, ...rest] = part.split(" "); + return ( + rest.length === 0 && + PROGRAM.test(program!) && + (sub === undefined || (sub.length <= SUBCOMMAND_LIMIT && SUBCOMMAND.test(sub))) + ); + }) + ); +} + +/** The programs a shell command line runs, in order and each once, as a record keeps them. */ +export function commandName(script: string): string { + return join(words(script).map(simple)); +} + +/** The program an argument vector runs, as a record keeps it, reading a shell's `-c` script as `commandName` does. */ +export function argvName(argv: string[]): string { + const program = argv[0]?.split("/").pop() ?? ""; + if (["sh", "bash", "zsh", "dash"].includes(program) && argv[1] === "-c" && argv[2] !== undefined) { + return commandName(argv[2]); + } + return join([simple(argv)]); +} + +function join(names: (string | null)[]): string { + const seen: string[] = []; + let length = 0; + for (const name of names) { + if (name === null || seen.includes(name)) continue; + const added = (seen.length ? 2 : 0) + name.length; + if (length + added > COMMAND_LIMIT) break; + seen.push(name); + length += added; + } + return seen.length ? seen.join("; ") : "?"; +} + +/** One simple command's program and subcommand, or null for one that names no program. */ +function simple(command: string[]): string | null { + let i = 0; + const assignment = (word: string) => /^[A-Za-z_][A-Za-z0-9_]*=/.test(word); + for (;;) { + const word = command[i]; + if (word === undefined) return null; + if (SKIPPED.has(word)) return null; + if (KEYWORDS.has(word) || assignment(word)) { + i++; + continue; + } + if (WRAPPERS.has(word.split("/").pop()!) && command[i + 1] !== undefined) { + i++; + while (command[i] !== undefined && /^(?:-.*|\d+[smhd]?|[A-Za-z_][A-Za-z0-9_]*=.*)$/.test(command[i]!)) i++; + continue; + } + break; + } + const base = command[i]!.split("/").pop()!; + const program = PROGRAM.test(base) ? base : "?"; + const sub = SUBCOMMANDS.has(program) ? command[i + 1] : undefined; + return sub !== undefined && sub.length <= SUBCOMMAND_LIMIT && SUBCOMMAND.test(sub) ? `${program} ${sub}` : program; +} + +/** + * The simple commands of a shell script, each as its words with quotes removed. Redirections and their targets, + * here-document bodies, and comments are dropped; text inside `$(…)` or backticks counts as commands of its own. + * This reads common scripts, not every shell grammar: what it misreads becomes a `?` or a word's own name. + */ +function words(script: string): string[][] { + const commands: string[][] = []; + let current: string[] = []; + let word: string | null = null; + const heredocs: string[] = []; + const endWord = () => { + if (word !== null) current.push(word); + word = null; + }; + const endCommand = () => { + endWord(); + if (current.length) commands.push(current); + current = []; + }; + let i = 0; + const n = script.length; + /** Read a quoted or plain word from `i` with no command structure, as a redirection's target. */ + const target = (): string => { + while (script[i] === " " || script[i] === "\t") i++; + let text = ""; + while (i < n && !/[\s;&|()<>`]/.test(script[i]!)) { + const c = script[i]!; + if (c === "'" || c === '"') { + const close = script.indexOf(c, i + 1); + const end = close === -1 ? n : close; + text += script.slice(i + 1, end); + i = end + 1; + } else if (c === "\\") { + text += script[i + 1] ?? ""; + i += 2; + } else { + text += c; + i++; + } + } + return text; + }; + while (i < n) { + const c = script[i]!; + if (c === "\n") { + endCommand(); + i++; + // A here-document's body runs to the line that is its delimiter alone. + while (heredocs.length && i < n) { + const eol = script.indexOf("\n", i); + const line = script.slice(i, eol === -1 ? n : eol); + i = eol === -1 ? n : eol + 1; + if (line.trim() === heredocs[0]) heredocs.shift(); + } + } else if (c === " " || c === "\t") { + endWord(); + i++; + } else if (c === "#" && word === null) { + while (i < n && script[i] !== "\n") i++; + } else if (c === "\\") { + if (script[i + 1] === "\n") i += 2; + else { + word = (word ?? "") + (script[i + 1] ?? ""); + i += 2; + } + } else if (c === "'") { + const close = script.indexOf("'", i + 1); + const end = close === -1 ? n : close; + word = (word ?? "") + script.slice(i + 1, end); + i = end + 1; + } else if (c === '"') { + let j = i + 1; + let text = ""; + while (j < n && script[j] !== '"') { + if (script[j] === "\\" && j + 1 < n) j++; + text += script[j]; + j++; + } + word = (word ?? "") + text; + i = j + 1; + } else if (c === "<" || c === ">" || (c === "&" && script[i + 1] === ">")) { + // A redirection: a file descriptor before it, its operator, and its target are no part of the command. + if (word !== null && /^\d+$/.test(word)) word = null; + endWord(); + let op = ""; + while (i < n && /[<>&|-]/.test(script[i]!) && op.length < 3) op += script[i++]; + if (op === "<<<") target(); + else if (op.startsWith("<<")) heredocs.push(target()); + else if (op.endsWith("&") && /\d|-/.test(script[i] ?? "")) { + while (/[\d-]/.test(script[i] ?? "")) i++; + } else target(); + } else if (c === "$" && script.startsWith("((", i + 1)) { + // Arithmetic runs no program. + const close = script.indexOf("))", i + 3); + word = (word ?? "") + "0"; + i = close === -1 ? n : close + 2; + } else if (c === "$" && script[i + 1] === "(") { + endCommand(); + i += 2; + } else if (";&|()`".includes(c)) { + endCommand(); + i++; + } else { + word = (word ?? "") + c; + i++; + } + } + endCommand(); + return commands; +} diff --git a/src/core/index.ts b/src/core/index.ts index 576fdad4..44a91e38 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -105,8 +105,8 @@ export { breakageKey, parseBreakage, } from "./breakage"; +export { COMMAND_LIMIT, argvName, commandName, isCommandName } from "./command"; export { - COMMAND_LIMIT, type GatedRun, type ModelSpend, type RecordUpload, @@ -123,7 +123,6 @@ export { NOTE_LIMIT, WAIT_KINDS, type WaitKind, - commandLine, parseRecord, parseUpload, } from "./record"; diff --git a/src/core/record.ts b/src/core/record.ts index 9ee7f0bf..cb353c91 100644 --- a/src/core/record.ts +++ b/src/core/record.ts @@ -1,4 +1,5 @@ import { type BreakageRecord, breakageKey, parseBreakage } from "./breakage"; +import { isCommandName } from "./command"; import { parser } from "./parse"; // The record of one agent session that `quest` sends to Kixel Code, which stores exactly this shape, and that @@ -34,18 +35,9 @@ export interface SessionWait { until: string | null; } -/** The longest command line a record keeps, in characters: enough to tell what ran, short of a heredoc's body. */ -export const COMMAND_LIMIT = 200; - -/** `text`'s first line, cut to `COMMAND_LIMIT` characters, as a record keeps a command. */ -export function commandLine(text: string): string { - const line = text.trimStart().split("\n", 1)[0]!.trimEnd(); - return line.length > COMMAND_LIMIT ? line.slice(0, COMMAND_LIMIT) : line; -} - /** One outermost `quest gate` run, its instants ISO 8601 UTC. */ export interface GatedRun { - /** Its command line, as `commandLine` keeps it. */ + /** The programs it ran, as `commandName` keeps them: no arguments. */ command: string; /** When it asked to be admitted. */ start: string; @@ -58,7 +50,7 @@ export interface GatedRun { /** One shell call a session's harness ran, its instants ISO 8601 UTC. */ export interface ShellCall { - /** Its command line, as `commandLine` keeps it. */ + /** The programs it ran, as `commandName` keeps them: no arguments. */ command: string; start: string; /** When it returned, or for one the harness ran in the background, when it was first seen finished. */ @@ -292,10 +284,10 @@ export function parseRecord(value: unknown): SessionRecord { return record; } -/** A command line no longer or wider than `commandLine` keeps. */ +/** Program names as `commandName` keeps them, so no record carries a command's arguments. */ function command(value: unknown, field: string): string { - const line = text(value, field); - return /[\r\n]/.test(line) || line.length > COMMAND_LIMIT ? fail(`${field} is not one short line`) : line; + const name = text(value, field); + return isCommandName(name) ? name : fail(`${field} is not program names alone`); } /** CPU milliseconds, or null. */ diff --git a/test/calls.test.ts b/test/calls.test.ts index bb8f0733..e88798e2 100644 --- a/test/calls.test.ts +++ b/test/calls.test.ts @@ -171,7 +171,7 @@ describe("shellGroups", () => { }); }); -test("the call log keeps whole lines and reads from a window's day", () => { +test("the call log keeps whole lines and reads from the day before a window's", () => { const env = { XDG_STATE_HOME: tempDir() }; const line: CallLine = { kind: "stop", @@ -184,7 +184,8 @@ test("the call log keeps whole lines and reads from a window's day", () => { }; logCall(line, env, Date.parse(line.at)); expect(readCalls(Date.parse("2026-11-14T20:00:00Z"), env)).toEqual([line]); - expect(readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toEqual([]); + expect(readCalls(Date.parse("2026-11-15T00:30:00Z"), env)).toEqual([line]); + expect(readCalls(Date.parse("2026-11-16T00:30:00Z"), env)).toEqual([]); logCall({ ...line, running: [""] }, env, Date.parse(line.at)); logCall(line, env, Date.parse(line.at)); expect(() => readCalls(Date.parse("2026-11-14T20:00:00Z"), env)).toThrow("is not a shell call quest logged"); @@ -215,3 +216,20 @@ test("reading a window also reads each earlier day a call in it began on", () => // The group they form began before the window, so no record of the window holds it. expect(shellGroups(read).map((g) => [g.start, g.calls.length])).toEqual([[Date.parse("2026-11-14T23:50:00Z"), 2]]); }); + +test("reading a window also reads the day before, for a background call no line in the window names", () => { + const env = { XDG_STATE_HOME: tempDir() }; + const sample = (at: string) => ({ at, cgroup: "/a.scope", usage: 0 }); + const caller = { harness: "claude", session: "s", agent: null }; + const log = (line: CallLine) => logCall(line, env, Date.parse(line.at)); + // A background call handed off the day before, finishing before the window's first turn end, which no longer + // lists it; a call in the window overlapped it. + log({ kind: "start", ...caller, call: "bg", command: "bg", ...sample("2026-11-14T23:50:00.000Z") }); + log({ kind: "end", ...caller, call: "bg", task: "b1", ...sample("2026-11-14T23:50:01.000Z") }); + log({ kind: "start", ...caller, call: "in", command: "in", ...sample("2026-11-15T00:01:00.000Z") }); + log({ kind: "end", ...caller, call: "in", task: null, ...sample("2026-11-15T00:01:30.000Z") }); + log({ kind: "stop", harness: "claude", session: "s", running: [], ...sample("2026-11-15T00:02:00.000Z") }); + const read = readCalls(Date.parse("2026-11-15T00:00:00Z"), env); + // One group, which began before the window, rather than one starting inside it. + expect(shellGroups(read).map((g) => [g.start, g.calls.length])).toEqual([[Date.parse("2026-11-14T23:50:00Z"), 2]]); +}); diff --git a/test/command.test.ts b/test/command.test.ts new file mode 100644 index 00000000..b37e86c7 --- /dev/null +++ b/test/command.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from "vitest"; + +import { COMMAND_LIMIT, argvName, commandName, isCommandName } from "../src/core"; + +test.each([ + ["cargo build --release", "cargo build"], + ["cd src && cargo test -p foo 2>&1 | tail -20", "cd; cargo test; tail"], + ["GH_TOKEN=ghp_secret gh pr view 1", "gh pr"], + ["curl -H 'Authorization: Bearer abcdef' https://x", "curl"], + ['git commit -m "fix; rm stuff && more"', "git commit"], + ["echo hunter2 > /tmp/x; cat < /tmp/x", "echo; cat"], + ["cat > x <<'EOF'\nsecret body; rm -rf /\nEOF\njust check", "cat; just check"], + ["env -i FOO=1 timeout 30s nice -n 10 bun test", "bun test"], + ["/usr/bin/time ./target/release/quest gate just test", "quest gate"], + ["for f in a b; do just check; done", "just check"], + ["if [[ -f x ]]; then make all; fi", "make all"], + ["x=$(git rev-parse HEAD)", "git rev-parse"], + ["git log abc123def", "git log"], + ["echo hunter", "echo"], + ["$EDITOR file", "?"], + ["# just a comment", "?"], + ["just check && just check", "just check"], +])("%j is kept as %j", (script, name) => { + expect(commandName(script)).toBe(name); + expect(isCommandName(name)).toBe(true); +}); + +test("an argument vector keeps its program and subcommand, reading a shell's script", () => { + expect(argvName(["cargo", "build", "--release"])).toBe("cargo build"); + expect(argvName(["/bin/sh", "-c", "cd x && bun test secret"])).toBe("cd; bun test"); + expect(argvName(["node", "-e", "spin()"])).toBe("node"); +}); + +test("a name stops at the limit, whole programs at a time", () => { + const name = commandName(Array.from({ length: 100 }, (_, i) => `p${i}`).join("; ")); + expect(name.length).toBeLessThanOrEqual(COMMAND_LIMIT); + expect(name.endsWith(";")).toBe(false); + expect(isCommandName(name)).toBe(true); +}); + +test.each(["", "cargo build --release", "gh pr view 1", "-f x", "a\nb", "cargo Build", "x;y"])( + "%j is no command name", + (name) => expect(isCommandName(name)).toBe(false), +); diff --git a/test/gate.test.ts b/test/gate.test.ts index 8bc9f7ab..14998dab 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -1555,7 +1555,8 @@ describe.skipIf(process.platform === "win32")("the run record", () => { expect(rest).toEqual([]); expect(line).toMatchObject({ kind: "gate", - command: `/bin/sh -c ${spin[2]}`, + // Its shell script runs no program but the shell's own builtins. + command: "?", sessions: [{ harness: "claude", session: "s1" }], directory: process.cwd(), }); diff --git a/test/hooks.test.ts b/test/hooks.test.ts index 05ced4ed..39196a54 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -283,7 +283,7 @@ test("the tool hooks sample each shell call's start and return, naming its sessi session: "s1", agent: "a1", call: "t1", - command: "cat > x <<'EOF'", + command: "cat", ...sample, }, { kind: "end", harness: "claude", session: "s1", agent: "a1", call: "t1", task: "b7", ...sample }, diff --git a/test/records.test.ts b/test/records.test.ts index bdc642fd..43d75395 100644 --- a/test/records.test.ts +++ b/test/records.test.ts @@ -152,7 +152,7 @@ describe("parseRecord", () => { "never started", ], [ - "a gated command over two lines", + "a gated command with its arguments", record({ gated: [ { @@ -163,21 +163,21 @@ describe("parseRecord", () => { end: "2026-11-06T10:30:00.000Z", cpu: 1200, }, - command: "a\nb", + command: "gh pr view 1", }, ], }), - "one short line", + "program names alone", ], ["a shell group without calls", record({ shell: [{ calls: [], cpu: 1, shared: false }] }), "calls"], [ - "a shell command longer than a record keeps", + "a shell command over two lines", record({ shell: [ { calls: [ { - command: "x".repeat(201), + command: "a\nb", start: "2026-11-06T10:31:00.000Z", end: "2026-11-06T10:31:01.000Z", }, @@ -187,7 +187,7 @@ describe("parseRecord", () => { }, ], }), - "one short line", + "program names alone", ], ["a quest that is not a path", record({ quest: "a.md" }), "quest path"], ])("refuses %s", (_, value, message) => { From b857fa33ede2f468565daeef2b3db53e745e58f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:49:29 -0800 Subject: [PATCH 06/13] fix(records): resume the outer command after a substitution, and match here-document delimiters exactly Co-Authored-By: Claude Opus 5.5 --- src/core/command.ts | 79 ++++++++++++++++++++++++++++++++++++-------- test/command.test.ts | 6 ++++ 2 files changed, 71 insertions(+), 14 deletions(-) diff --git a/src/core/command.ts b/src/core/command.ts index 71c9cad5..cff86f1f 100644 --- a/src/core/command.ts +++ b/src/core/command.ts @@ -128,7 +128,8 @@ function words(script: string): string[][] { const commands: string[][] = []; let current: string[] = []; let word: string | null = null; - const heredocs: string[] = []; + /** Pending here-documents' delimiters, and whether `<<-` lets tabs lead them. */ + const heredocs: { delimiter: string; tabs: boolean }[] = []; const endWord = () => { if (word !== null) current.push(word); word = null; @@ -140,6 +141,16 @@ function words(script: string): string[][] { }; let i = 0; const n = script.length; + /** + * Read the command substitution whose text starts at `from` and ends at `close`'s index, as `script.indexOf` + * finds it, as commands of its own; the word it sits in goes on, its output standing as one character. + */ + const substitute = (from: number, close: number) => { + const end = close === -1 ? n : close; + commands.push(...words(script.slice(from, end))); + word = (word ?? "") + "0"; + i = end + 1; + }; /** Read a quoted or plain word from `i` with no command structure, as a redirection's target. */ const target = (): string => { while (script[i] === " " || script[i] === "\t") i++; @@ -171,7 +182,8 @@ function words(script: string): string[][] { const eol = script.indexOf("\n", i); const line = script.slice(i, eol === -1 ? n : eol); i = eol === -1 ? n : eol + 1; - if (line.trim() === heredocs[0]) heredocs.shift(); + const { delimiter, tabs } = heredocs[0]!; + if ((tabs ? line.replace(/^\t+/, "") : line) === delimiter) heredocs.shift(); } } else if (c === " " || c === "\t") { endWord(); @@ -190,15 +202,20 @@ function words(script: string): string[][] { word = (word ?? "") + script.slice(i + 1, end); i = end + 1; } else if (c === '"') { - let j = i + 1; - let text = ""; - while (j < n && script[j] !== '"') { - if (script[j] === "\\" && j + 1 < n) j++; - text += script[j]; - j++; + // Literal text stays data; a substitution inside runs commands. + word ??= ""; + i++; + while (i < n && script[i] !== '"') { + if (script[i] === "$" && script[i + 1] === "(" && script[i + 2] !== "(") + substitute(i + 2, closing(script, i + 2)); + else if (script[i] === "`") substitute(i + 1, backtick(script, i + 1)); + else { + if (script[i] === "\\" && i + 1 < n) i++; + word += script[i]; + i++; + } } - word = (word ?? "") + text; - i = j + 1; + i++; } else if (c === "<" || c === ">" || (c === "&" && script[i + 1] === ">")) { // A redirection: a file descriptor before it, its operator, and its target are no part of the command. if (word !== null && /^\d+$/.test(word)) word = null; @@ -206,7 +223,7 @@ function words(script: string): string[][] { let op = ""; while (i < n && /[<>&|-]/.test(script[i]!) && op.length < 3) op += script[i++]; if (op === "<<<") target(); - else if (op.startsWith("<<")) heredocs.push(target()); + else if (op.startsWith("<<")) heredocs.push({ tabs: op === "<<-", delimiter: target() }); else if (op.endsWith("&") && /\d|-/.test(script[i] ?? "")) { while (/[\d-]/.test(script[i] ?? "")) i++; } else target(); @@ -216,9 +233,10 @@ function words(script: string): string[][] { word = (word ?? "") + "0"; i = close === -1 ? n : close + 2; } else if (c === "$" && script[i + 1] === "(") { - endCommand(); - i += 2; - } else if (";&|()`".includes(c)) { + substitute(i + 2, closing(script, i + 2)); + } else if (c === "`") { + substitute(i + 1, backtick(script, i + 1)); + } else if (";&|()".includes(c)) { endCommand(); i++; } else { @@ -229,3 +247,36 @@ function words(script: string): string[][] { endCommand(); return commands; } + +/** The index of the `)` closing the substitution whose text starts at `from`, past nested ones and quotes, or -1. */ +function closing(script: string, from: number): number { + let depth = 1; + for (let i = from; i < script.length; i++) { + const c = script[i]!; + if (c === "\\") i++; + else if (c === "'") { + const close = script.indexOf("'", i + 1); + if (close === -1) return -1; + i = close; + } else if (c === '"') { + for (i++; i < script.length && script[i] !== '"'; i++) { + if (script[i] === "\\") i++; + else if (script[i] === "$" && script[i + 1] === "(") { + i = closing(script, i + 2); + if (i === -1) return -1; + } + } + } else if (c === "(") depth++; + else if (c === ")" && --depth === 0) return i; + } + return -1; +} + +/** The index of the backtick closing the substitution whose text starts at `from`, or -1. */ +function backtick(script: string, from: number): number { + for (let i = from; i < script.length; i++) { + if (script[i] === "\\") i++; + else if (script[i] === "`") return i; + } + return -1; +} diff --git a/test/command.test.ts b/test/command.test.ts index b37e86c7..99973181 100644 --- a/test/command.test.ts +++ b/test/command.test.ts @@ -20,6 +20,12 @@ test.each([ ["$EDITOR file", "?"], ["# just a comment", "?"], ["just check && just check", "just check"], + ["printf '%s' $(date +%s) hunter2", "date; printf"], + ["printf %s `date` hunter2; ls", "date; printf; ls"], + ['output="$(cargo check)"; echo "$output"', "cargo check; echo"], + ['echo "a $(git log --format="%H (x)") b"', "git log; echo"], + ["cat <<'EOF'\n EOF\nhunter2\nEOF\nls", "cat; ls"], + ["cat <<-EOF\n\thunter2\n\tEOF\nls", "cat; ls"], ])("%j is kept as %j", (script, name) => { expect(commandName(script)).toBe(name); expect(isCommandName(name)).toBe(true); From 44faecbf7d9e8de53da46eb3073c52bf271be1e6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:55:05 -0800 Subject: [PATCH 07/13] fix(records): count a program only where the host can run it, and skip here-documents finding a substitution's end Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++-- src/cli/calls.ts | 11 +++--- src/cli/gate.ts | 3 +- src/cli/hooks.ts | 9 ++++- src/cli/program.ts | 18 ++++++++++ src/core/command.ts | 82 ++++++++++++++++++++++++++++++++++++++------ src/core/index.ts | 2 +- test/command.test.ts | 33 +++++++++++++++--- test/hooks.test.ts | 8 ++--- 9 files changed, 144 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index b7a92f27..f6b06f60 100644 --- a/README.md +++ b/README.md @@ -1839,8 +1839,10 @@ latest upload, since one sender sends a session's uploads in order. The type is never arguments, assignments, redirections, or here-document bodies, since a command line can carry a secret and records leave the host. Programs such as `git`, `cargo`, `just`, `nix`, and `npm` keep their subcommand; - any other's first argument is data and stays. A program written as a - variable is `?`. + any other's first argument is data and stays. A program counts only as a + shell builtin or one the host can run, on `PATH` or by its path, so a + variable, or a word Quest's reading of the script mistook for a program, + is `?`. - `split` is where the rest of its time went, as the next section describes; null until every step is labeled, and for a session on another host. diff --git a/src/cli/calls.ts b/src/cli/calls.ts index ec866649..4952b117 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -4,7 +4,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path/posix"; -import { type ShellCall, type ShellGroup, commandName } from "../core"; +import { type Runs, type ShellCall, type ShellGroup, commandName } from "../core"; import { cgroupUsage, members } from "./supervise"; import { appendLog, logDays, logDirectory, readDays } from "./waits"; @@ -42,9 +42,12 @@ export function logCall(line: CallLine, env: NodeJS.ProcessEnv = process.env, no appendLog(callsDirectory(env), line, now, "shell call"); } -/** A start line for `command`, as a record keeps it: the programs it runs, without arguments. */ -export function startLine(caller: Caller, call: string, command: string, sample: Sample): CallLine { - return { kind: "start", ...caller, call, command: commandName(command), ...sample }; +/** + * A start line for `command`, as a record keeps it: the programs it runs, without arguments, each one `runs` says + * the host can run. + */ +export function startLine(caller: Caller, call: string, command: string, runs: Runs, sample: Sample): CallLine { + return { kind: "start", ...caller, call, command: commandName(command, runs), ...sample }; } /** diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 069702d9..c96b0753 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -25,6 +25,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { type Cancellation, cancelled, follow, forwards, GROUP, interrupts, received, requested } from "./interrupt"; import { stubbed } from "./job"; import { argvName } from "../core"; +import { runs } from "./program"; import { envSessions, logWait } from "./waits"; import { cgroupUsage, measuredRun, questScope, scopeGroup, startTime, systemdScopes } from "./supervise"; @@ -930,7 +931,7 @@ export async function gate(command: string[], options: GateOptions = {}): Promis logWait( { kind: "gate", - command: argvName(command), + command: argvName(command, runs(process.env, directory)), start: iso(started), admitted: observed.admitted === undefined ? null : iso(observed.admitted), end: iso(Date.now()), diff --git a/src/cli/hooks.ts b/src/cli/hooks.ts index 020f3e1d..f85c4cab 100644 --- a/src/cli/hooks.ts +++ b/src/cli/hooks.ts @@ -16,6 +16,7 @@ import { CONTEXT } from "./skills"; import { HOOK_EVENTS } from "./worktree-hook"; import { queue } from "./send"; import { type Sample, logCall, sessionSample, startLine } from "./calls"; +import { runs } from "./program"; /** The command the rule hooks run. */ export const HOOK_COMMAND = "quest hook"; @@ -344,7 +345,13 @@ export function hook(input: unknown, deps: HookDependencies = { env: process.env if (rules.length === 0) { if (value.tool_name === "Bash" && typeof value.tool_input.command === "string") logCall( - startLine(caller(value), callId(value), value.tool_input.command, sample(deps, value)), + startLine( + caller(value), + callId(value), + value.tool_input.command, + runs(deps.env, value.cwd), + sample(deps, value), + ), deps.env, ); return ""; diff --git a/src/cli/program.ts b/src/cli/program.ts index 51327868..0ff13eaa 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -66,6 +66,24 @@ export function which(name: string, env: NodeJS.ProcessEnv): string | undefined return undefined; } +/** + * Whether a command's program word runs something on this host: a name on PATH, or a path, resolved from `cwd`, to + * a launchable file. Each answer is cached, since one script often names a program many times. + */ +export function runs(env: NodeJS.ProcessEnv, cwd: string): (program: string) => boolean { + const cache = new Map(); + return (program) => { + let found = cache.get(program); + if (found === undefined) { + found = program.includes("/") + ? launchable(posix.resolve(cwd, program), process.platform) + : which(program, env) !== undefined; + cache.set(program, found); + } + return found; + }; +} + /** `which`, or an error naming the missing program. */ export function requireProgram(name: string, env: NodeJS.ProcessEnv): string { const found = which(name, env); diff --git a/src/core/command.ts b/src/core/command.ts index cff86f1f..06a6d016 100644 --- a/src/core/command.ts +++ b/src/core/command.ts @@ -2,7 +2,9 @@ // as `cd; cargo build; tail`. Arguments, assignments, redirections, and quoted text stay on the host, since a // command line can carry a secret, such as an inline token, and records leave it. That is still enough for the // mistake audit to tell a cheap command from a heavy one: `cargo check` from `cargo build`, `just check` from `just -// test`. +// test`. A shell script is no easy grammar, so the parse below is not what keeps a secret on the host: a program +// counts only where it is a shell builtin or the host can run it, as the caller's `runs` decides, and any other +// word read as one, such as a password a misread here-document gave, becomes `?`. /** The longest name a record keeps, in characters; names past it are left out. */ export const COMMAND_LIMIT = 200; @@ -44,6 +46,39 @@ const SUBCOMMANDS = new Set([ "yarn", ]); +/** Shell builtins, which name no file on PATH yet run as programs. */ +const BUILTINS = new Set([ + ".", + ":", + "alias", + "cd", + "declare", + "echo", + "eval", + "exit", + "export", + "local", + "popd", + "printf", + "pushd", + "pwd", + "read", + "return", + "set", + "shift", + "source", + "test", + "trap", + "type", + "ulimit", + "umask", + "unset", + "wait", +]); + +/** Whether a command's program word, a name or a path, runs on the host that ran it. */ +export type Runs = (program: string) => boolean; + /** Words that open a compound command or a wrapper of the next one, skipped to reach the program it runs. */ const KEYWORDS = new Set(["if", "then", "else", "elif", "do", "while", "until", "!", "{", "}"]); /** Words that run the command after them, with flags and a number, such as a priority or timeout, between. */ @@ -68,17 +103,17 @@ export function isCommandName(name: string): boolean { } /** The programs a shell command line runs, in order and each once, as a record keeps them. */ -export function commandName(script: string): string { - return join(words(script).map(simple)); +export function commandName(script: string, runs: Runs): string { + return join(words(script).map((command) => simple(command, runs))); } /** The program an argument vector runs, as a record keeps it, reading a shell's `-c` script as `commandName` does. */ -export function argvName(argv: string[]): string { +export function argvName(argv: string[], runs: Runs): string { const program = argv[0]?.split("/").pop() ?? ""; if (["sh", "bash", "zsh", "dash"].includes(program) && argv[1] === "-c" && argv[2] !== undefined) { - return commandName(argv[2]); + return commandName(argv[2], runs); } - return join([simple(argv)]); + return join([simple(argv, runs)]); } function join(names: (string | null)[]): string { @@ -95,7 +130,7 @@ function join(names: (string | null)[]): string { } /** One simple command's program and subcommand, or null for one that names no program. */ -function simple(command: string[]): string | null { +function simple(command: string[], runs: Runs): string | null { let i = 0; const assignment = (word: string) => /^[A-Za-z_][A-Za-z0-9_]*=/.test(word); for (;;) { @@ -113,8 +148,9 @@ function simple(command: string[]): string | null { } break; } - const base = command[i]!.split("/").pop()!; - const program = PROGRAM.test(base) ? base : "?"; + const first = command[i]!; + const base = first.split("/").pop()!; + const program = PROGRAM.test(base) && ((!first.includes("/") && BUILTINS.has(first)) || runs(first)) ? base : "?"; const sub = SUBCOMMANDS.has(program) ? command[i + 1] : undefined; return sub !== undefined && sub.length <= SUBCOMMAND_LIMIT && SUBCOMMAND.test(sub) ? `${program} ${sub}` : program; } @@ -248,11 +284,16 @@ function words(script: string): string[][] { return commands; } -/** The index of the `)` closing the substitution whose text starts at `from`, past nested ones and quotes, or -1. */ +/** + * The index of the `)` closing the substitution whose text starts at `from`, past nested ones, quotes, comments, and + * here-document bodies, where a parenthesis is data; or -1. + */ function closing(script: string, from: number): number { let depth = 1; + const heredocs: { delimiter: string; tabs: boolean }[] = []; for (let i = from; i < script.length; i++) { const c = script[i]!; + const wordStart = i === from || /[\s;&|()]/.test(script[i - 1]!); if (c === "\\") i++; else if (c === "'") { const close = script.indexOf("'", i + 1); @@ -266,6 +307,27 @@ function closing(script: string, from: number): number { if (i === -1) return -1; } } + } else if (c === "#" && wordStart) { + while (i + 1 < script.length && script[i + 1] !== "\n") i++; + } else if (c === "<" && script.startsWith("<<", i) && script[i + 2] !== "<") { + const tabs = script[i + 2] === "-"; + const match = /^[ \t]*(?:'([^']*)'|"([^"]*)"|([^\s;&|()<>]+))/.exec(script.slice(i + (tabs ? 3 : 2))); + if (match) { + heredocs.push({ delimiter: (match[1] ?? match[2] ?? match[3]!).replace(/\\/g, ""), tabs }); + i += (tabs ? 3 : 2) + match[0].length - 1; + } + } else if (c === "\n" && heredocs.length) { + // Skip each pending body, up to and including its delimiter's line. + let at = i + 1; + while (heredocs.length && at < script.length) { + const eol = script.indexOf("\n", at); + const line = script.slice(at, eol === -1 ? script.length : eol); + at = eol === -1 ? script.length : eol + 1; + const { delimiter, tabs } = heredocs[0]!; + if ((tabs ? line.replace(/^\t+/, "") : line) === delimiter) heredocs.shift(); + } + if (heredocs.length) return -1; + i = at - 1; } else if (c === "(") depth++; else if (c === ")" && --depth === 0) return i; } diff --git a/src/core/index.ts b/src/core/index.ts index 44a91e38..ab46a0fb 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -105,7 +105,7 @@ export { breakageKey, parseBreakage, } from "./breakage"; -export { COMMAND_LIMIT, argvName, commandName, isCommandName } from "./command"; +export { COMMAND_LIMIT, type Runs, argvName, commandName, isCommandName } from "./command"; export { type GatedRun, type ModelSpend, diff --git a/test/command.test.ts b/test/command.test.ts index 99973181..b28c2c5a 100644 --- a/test/command.test.ts +++ b/test/command.test.ts @@ -1,6 +1,15 @@ import { expect, test } from "vitest"; import { COMMAND_LIMIT, argvName, commandName, isCommandName } from "../src/core"; +import { runs } from "../src/cli/program"; + +/** The programs a host in these tests can run, by name or path. */ +const installed = new Set( + "bun cargo cat curl date gh git just ls make node quest tail /usr/bin/time ./target/release/quest /bin/sh".split( + " ", + ), +); +const host = (program: string) => installed.has(program); test.each([ ["cargo build --release", "cargo build"], @@ -26,19 +35,24 @@ test.each([ ['echo "a $(git log --format="%H (x)") b"', "git log; echo"], ["cat <<'EOF'\n EOF\nhunter2\nEOF\nls", "cat; ls"], ["cat <<-EOF\n\thunter2\n\tEOF\nls", "cat; ls"], + ["printf '%s' $(cat <<'EOF'\n)\nhunter2\nEOF\n)", "cat; printf"], + ["x=$(ls # a ) comment\n)", "ls"], + // A word misread as a program, or one the host cannot run, stays a `?`. + ["hunter2; gh pr view", "?; gh pr"], + ["./hunter2 x", "?"], ])("%j is kept as %j", (script, name) => { - expect(commandName(script)).toBe(name); + expect(commandName(script, host)).toBe(name); expect(isCommandName(name)).toBe(true); }); test("an argument vector keeps its program and subcommand, reading a shell's script", () => { - expect(argvName(["cargo", "build", "--release"])).toBe("cargo build"); - expect(argvName(["/bin/sh", "-c", "cd x && bun test secret"])).toBe("cd; bun test"); - expect(argvName(["node", "-e", "spin()"])).toBe("node"); + expect(argvName(["cargo", "build", "--release"], host)).toBe("cargo build"); + expect(argvName(["/bin/sh", "-c", "cd x && bun test secret"], host)).toBe("cd; bun test"); + expect(argvName(["node", "-e", "spin()"], host)).toBe("node"); }); test("a name stops at the limit, whole programs at a time", () => { - const name = commandName(Array.from({ length: 100 }, (_, i) => `p${i}`).join("; ")); + const name = commandName(Array.from({ length: 100 }, (_, i) => `p${i}`).join("; "), () => true); expect(name.length).toBeLessThanOrEqual(COMMAND_LIMIT); expect(name.endsWith(";")).toBe(false); expect(isCommandName(name)).toBe(true); @@ -48,3 +62,12 @@ test.each(["", "cargo build --release", "gh pr view 1", "-f x", "a\nb", "cargo B "%j is no command name", (name) => expect(isCommandName(name)).toBe(false), ); + +test("a host runs a name on PATH or a path to a launchable file, from its directory", () => { + const on = runs({ PATH: "/bin:/usr/bin" }, "/"); + expect(on("sh")).toBe(true); + expect(on("bin/sh")).toBe(true); + expect(on("hunter2")).toBe(false); + expect(on("./hunter2")).toBe(false); + expect(on("etc/passwd")).toBe(false); +}); diff --git a/test/hooks.test.ts b/test/hooks.test.ts index 39196a54..ac7e5324 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -223,7 +223,7 @@ test("SessionStart adds nothing where the repository does not use Quest", () => /** Hook dependencies writing to a fresh state directory, sampling a fixed counter. */ function sampled(usage = 5_000) { - const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir() }; + const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir(), PATH: "/bin:/usr/bin" }; const sample = () => ({ at: "2026-11-15T10:00:00.000Z", cgroup: "/app.slice/agent.scope", usage }); return { env, sample, lines: () => readCalls(Date.parse("2026-11-15T00:00:00Z"), env) }; } @@ -268,9 +268,9 @@ test("the tool hooks sample each shell call's start and return, naming its sessi ), ).toBe(""); const codex = { cwd, session_id: "c1", turn_id: "u1", tool_use_id: "t2", tool_name: "Bash" }; - hook({ ...codex, hook_event_name: "PreToolUse", tool_input: { command: "cargo test" } }, deps); + hook({ ...codex, hook_event_name: "PreToolUse", tool_input: { command: "ls -la hunter2" } }, deps); hook( - { ...codex, hook_event_name: "PostToolUse", tool_input: { command: "cargo test" }, tool_response: "ok" }, + { ...codex, hook_event_name: "PostToolUse", tool_input: { command: "ls -la hunter2" }, tool_response: "ok" }, deps, ); // Edits are no shell calls. @@ -287,7 +287,7 @@ test("the tool hooks sample each shell call's start and return, naming its sessi ...sample, }, { kind: "end", harness: "claude", session: "s1", agent: "a1", call: "t1", task: "b7", ...sample }, - { kind: "start", harness: "codex", session: "c1", agent: null, call: "t2", command: "cargo test", ...sample }, + { kind: "start", harness: "codex", session: "c1", agent: null, call: "t2", command: "ls", ...sample }, { kind: "end", harness: "codex", session: "c1", agent: null, call: "t2", task: null, ...sample }, ]); expect(() => From b4d7dd39faaa2d1a24cbad5e49ed4d1b9ee2d561 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 10:59:54 -0800 Subject: [PATCH 08/13] fix(records): keep subcommands only from a fixed list, follow cd for relative programs, and skip here-strings Co-Authored-By: Claude Opus 5.5 --- README.md | 11 ++++----- src/cli/program.ts | 18 +++++++-------- src/core/command.ts | 53 +++++++++++++++++++++++++++++++++----------- test/command.test.ts | 35 +++++++++++++++++++++-------- 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index f6b06f60..9c981345 100644 --- a/README.md +++ b/README.md @@ -1838,11 +1838,12 @@ latest upload, since one sender sends a session's uploads in order. The type is the subcommand of those that take one, such as `cd; cargo build; tail`: never arguments, assignments, redirections, or here-document bodies, since a command line can carry a secret and records leave the host. Programs - such as `git`, `cargo`, `just`, `nix`, and `npm` keep their subcommand; - any other's first argument is data and stays. A program counts only as a - shell builtin or one the host can run, on `PATH` or by its path, so a - variable, or a word Quest's reading of the script mistook for a program, - is `?`. + such as `git`, `cargo`, `just`, `nix`, and `npm` keep their subcommand + when it is one of a fixed list of common ones, such as `build`, `check`, + and `test`; any other's first argument is data and stays. A program counts + only as a shell builtin or one the host can run, on `PATH` or by its path, + so a variable, or a word Quest's reading of the script mistook for a + program, is `?`. - `split` is where the rest of its time went, as the next section describes; null until every step is labeled, and for a session on another host. diff --git a/src/cli/program.ts b/src/cli/program.ts index 0ff13eaa..e6d53d9e 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -67,18 +67,18 @@ export function which(name: string, env: NodeJS.ProcessEnv): string | undefined } /** - * Whether a command's program word runs something on this host: a name on PATH, or a path, resolved from `cwd`, to - * a launchable file. Each answer is cached, since one script often names a program many times. + * Whether a command's program word runs something on this host: a name on PATH, or a path, resolved from `cwd` and + * then each directory the command changed to, to a launchable file. Each answer is cached, since one script often + * names a program many times. */ -export function runs(env: NodeJS.ProcessEnv, cwd: string): (program: string) => boolean { +export function runs(env: NodeJS.ProcessEnv, cwd: string): (program: string, cd: string[]) => boolean { const cache = new Map(); - return (program) => { - let found = cache.get(program); + return (program, cd) => { + const path = program.includes("/") ? posix.resolve(cwd, ...cd, program) : program; + let found = cache.get(path); if (found === undefined) { - found = program.includes("/") - ? launchable(posix.resolve(cwd, program), process.platform) - : which(program, env) !== undefined; - cache.set(program, found); + found = program.includes("/") ? launchable(path, process.platform) : which(program, env) !== undefined; + cache.set(path, found); } return found; }; diff --git a/src/core/command.ts b/src/core/command.ts index 06a6d016..6f50cbd9 100644 --- a/src/core/command.ts +++ b/src/core/command.ts @@ -3,17 +3,28 @@ // command line can carry a secret, such as an inline token, and records leave it. That is still enough for the // mistake audit to tell a cheap command from a heavy one: `cargo check` from `cargo build`, `just check` from `just // test`. A shell script is no easy grammar, so the parse below is not what keeps a secret on the host: a program -// counts only where it is a shell builtin or the host can run it, as the caller's `runs` decides, and any other -// word read as one, such as a password a misread here-document gave, becomes `?`. +// counts only where it is a shell builtin or the host can run it, as the caller's `runs` decides, and a subcommand +// only where it is one of the fixed `VERBS`. Any other word read as one, such as a password a misread +// here-document or quote gave, becomes `?` or is left out, so a misreading costs the audit detail, never a secret. /** The longest name a record keeps, in characters; names past it are left out. */ export const COMMAND_LIMIT = 200; /** A program's name: a path's last part, short and of plain characters, or `?` for one written otherwise. */ const PROGRAM = /^(?:[A-Za-z0-9._+][A-Za-z0-9._+-]{0,31}|\?)$/; -/** A subcommand: lowercase words joined by hyphens, as `build` or `merge-base`, never a value such as a token. */ -const SUBCOMMAND = /^[a-z]{1,24}(?:-[a-z]{1,24})*$/; -const SUBCOMMAND_LIMIT = 24; +/** + * The subcommands a record keeps, a fixed list rather than any word's shape, so no data read as one leaves the host: + * those of the build tools, package managers, and forge and service tools agents run, and the common names of + * `just` and `make` recipes. Any other is left out, keeping only its program. + */ +const VERBS = new Set( + `add all api apply audit auth bench blame branch build check checkout cherry-pick ci clean clippy clone commit + compose config coverage create daemon delete deploy describe dev develop diff disable doc enable eval exec fetch + flake fmt format gate gc generate get grep init install issue lint lock log logs merge metadata mod mv nextest + outdated pr ps publish pull push rebase release remove repo reset restart restore rev-parse review rm run search serve + shell show start stash status stop switch sync tag test tidy tree typecheck uninstall update upgrade usage vet + watch workflow worktree x`.split(/\s+/), +); /** * Programs whose first argument names what they do, kept as its subcommand. Any other program's first argument is @@ -76,8 +87,11 @@ const BUILTINS = new Set([ "wait", ]); -/** Whether a command's program word, a name or a path, runs on the host that ran it. */ -export type Runs = (program: string) => boolean; +/** + * Whether a command's program word, a name or a path, runs on the host that ran it, a relative path resolved from + * the directories the script changed to before it, in order, from where it started. + */ +export type Runs = (program: string, cd: string[]) => boolean; /** Words that open a compound command or a wrapper of the next one, skipped to reach the program it runs. */ const KEYWORDS = new Set(["if", "then", "else", "elif", "do", "while", "until", "!", "{", "}"]); @@ -96,7 +110,7 @@ export function isCommandName(name: string): boolean { return ( rest.length === 0 && PROGRAM.test(program!) && - (sub === undefined || (sub.length <= SUBCOMMAND_LIMIT && SUBCOMMAND.test(sub))) + (sub === undefined || (SUBCOMMANDS.has(program!) && VERBS.has(sub))) ); }) ); @@ -104,16 +118,26 @@ export function isCommandName(name: string): boolean { /** The programs a shell command line runs, in order and each once, as a record keeps them. */ export function commandName(script: string, runs: Runs): string { - return join(words(script).map((command) => simple(command, runs))); + // Each literal `cd` the script ran so far, in order. A `cd` in a subshell or a failed one misleads this, which + // at worst records a `?` for a program that did run. + const cd: string[] = []; + return join( + words(script).map((command) => { + if (command[0] === "cd" && command[1] !== undefined && command[1] !== "-" && !command[1].includes("$")) + cd.push(command[1]); + return simple(command, (program) => runs(program, cd)); + }), + ); } /** The program an argument vector runs, as a record keeps it, reading a shell's `-c` script as `commandName` does. */ export function argvName(argv: string[], runs: Runs): string { + const run = (program: string) => runs(program, []); const program = argv[0]?.split("/").pop() ?? ""; if (["sh", "bash", "zsh", "dash"].includes(program) && argv[1] === "-c" && argv[2] !== undefined) { return commandName(argv[2], runs); } - return join([simple(argv, runs)]); + return join([simple(argv, run)]); } function join(names: (string | null)[]): string { @@ -130,7 +154,7 @@ function join(names: (string | null)[]): string { } /** One simple command's program and subcommand, or null for one that names no program. */ -function simple(command: string[], runs: Runs): string | null { +function simple(command: string[], runs: (program: string) => boolean): string | null { let i = 0; const assignment = (word: string) => /^[A-Za-z_][A-Za-z0-9_]*=/.test(word); for (;;) { @@ -152,7 +176,7 @@ function simple(command: string[], runs: Runs): string | null { const base = first.split("/").pop()!; const program = PROGRAM.test(base) && ((!first.includes("/") && BUILTINS.has(first)) || runs(first)) ? base : "?"; const sub = SUBCOMMANDS.has(program) ? command[i + 1] : undefined; - return sub !== undefined && sub.length <= SUBCOMMAND_LIMIT && SUBCOMMAND.test(sub) ? `${program} ${sub}` : program; + return sub !== undefined && VERBS.has(sub) ? `${program} ${sub}` : program; } /** @@ -309,7 +333,10 @@ function closing(script: string, from: number): number { } } else if (c === "#" && wordStart) { while (i + 1 < script.length && script[i + 1] !== "\n") i++; - } else if (c === "<" && script.startsWith("<<", i) && script[i + 2] !== "<") { + } else if (script.startsWith("<<<", i)) { + // A here-string, whose word follows on its line, opens no here-document. + i += 2; + } else if (c === "<" && script.startsWith("<<", i)) { const tabs = script[i + 2] === "-"; const match = /^[ \t]*(?:'([^']*)'|"([^"]*)"|([^\s;&|()<>]+))/.exec(script.slice(i + (tabs ? 3 : 2))); if (match) { diff --git a/test/command.test.ts b/test/command.test.ts index b28c2c5a..8949fb01 100644 --- a/test/command.test.ts +++ b/test/command.test.ts @@ -1,3 +1,5 @@ +import { posix } from "node:path"; + import { expect, test } from "vitest"; import { COMMAND_LIMIT, argvName, commandName, isCommandName } from "../src/core"; @@ -5,11 +7,10 @@ import { runs } from "../src/cli/program"; /** The programs a host in these tests can run, by name or path. */ const installed = new Set( - "bun cargo cat curl date gh git just ls make node quest tail /usr/bin/time ./target/release/quest /bin/sh".split( - " ", - ), + "bun cargo cat curl date gh git just ls make node quest tail /usr/bin/time target/release/quest /bin/sh".split(" "), ); -const host = (program: string) => installed.has(program); +const host = (program: string, cd: string[]) => + installed.has(program.includes("/") ? posix.join(...cd, program) : program); test.each([ ["cargo build --release", "cargo build"], @@ -40,6 +41,12 @@ test.each([ // A word misread as a program, or one the host cannot run, stays a `?`. ["hunter2; gh pr view", "?; gh pr"], ["./hunter2 x", "?"], + ['echo "$(cat <<< hi\n)" "hello; git password"', "cat; echo"], + // A subcommand outside the fixed list is left out, as a password a misread quote would make one. + ["git password", "git"], + ["just my-recipe", "just"], + ['output="$(cat <<< foo\n)"; cargo check', "cat; cargo check"], + ["cd target && release/quest gate", "cd; quest gate"], ])("%j is kept as %j", (script, name) => { expect(commandName(script, host)).toBe(name); expect(isCommandName(name)).toBe(true); @@ -58,16 +65,26 @@ test("a name stops at the limit, whole programs at a time", () => { expect(isCommandName(name)).toBe(true); }); -test.each(["", "cargo build --release", "gh pr view 1", "-f x", "a\nb", "cargo Build", "x;y"])( - "%j is no command name", - (name) => expect(isCommandName(name)).toBe(false), -); +test.each([ + "", + "cargo build --release", + "gh pr view 1", + "-f x", + "a\nb", + "cargo Build", + "x;y", + "git password", + "ls test", +])("%j is no command name", (name) => expect(isCommandName(name)).toBe(false)); test("a host runs a name on PATH or a path to a launchable file, from its directory", () => { - const on = runs({ PATH: "/bin:/usr/bin" }, "/"); + const run = runs({ PATH: "/bin:/usr/bin" }, "/"); + const on = (program: string, cd: string[] = []) => run(program, cd); expect(on("sh")).toBe(true); expect(on("bin/sh")).toBe(true); expect(on("hunter2")).toBe(false); expect(on("./hunter2")).toBe(false); expect(on("etc/passwd")).toBe(false); + expect(on("./sh", ["usr", "bin"])).toBe(true); + expect(on("./sh", ["usr"])).toBe(false); }); From aa34fbcc8ba5532c01987eb2979719f6284c86cd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 11:10:34 -0800 Subject: [PATCH 09/13] fix(records): refuse a cgroup shared with a tree whose parent runs elsewhere, and read Windows program paths Co-Authored-By: Claude Opus 5.5 --- src/cli/calls.ts | 19 +++++++++++++++++-- src/cli/program.ts | 7 +++++-- src/core/command.ts | 13 +++++++++---- test/calls.test.ts | 7 +++++++ test/command.test.ts | 40 +++++++++++++++++++++++++++++----------- 5 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/cli/calls.ts b/src/cli/calls.ts index 4952b117..27e27f3b 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -134,7 +134,8 @@ export interface Host { * Such a cgroup holds one tree of processes, whose top is the harness: the hook's topmost ancestor in the cgroup, from * which every other branch hangs, as when an app starts each agent session in a scope of its own. A terminal's scope * fails, since its top is the terminal's shell, as does one shared with other work. A process whose parent left, as - * a daemonized one's does, is the session's leftover and counts. + * a daemonized one's does, is the session's leftover and counts where init or a systemd manager adopted it; one + * whose parent runs outside the cgroup is other work. */ export function sessionSample(now: number, host: Host): Sample { const at = new Date(now).toISOString(); @@ -161,15 +162,29 @@ export function sessionSample(now: number, host: Host): Sample { if (!chain.has(host.self) || !harnessIs(host, top)) return none; for (const pid of inside) { if (chain.has(pid)) continue; - // The chain member this process hangs from, or undefined for a tree whose root's parent left the cgroup. + // The chain member this process hangs from, else its tree's root's parent outside the cgroup, or undefined. let at: number | undefined = parent(pid); while (at !== undefined && inside.has(at) && !chain.has(at)) at = parent(at); if (at !== undefined && chain.has(at) && at !== top) return none; + // A tree outside the harness's counts only as a daemon the session left, which init or the user's service + // manager, as the reaper of orphans, adopted. A live parent elsewhere means other work shares the cgroup. + if (at !== undefined && !chain.has(at) && !adopted(host.proc, at)) return none; } const usage = cgroupUsage(group); return usage === null ? none : { at, cgroup: path, usage }; } +/** Whether `pid` reaps orphans: init, or a systemd service manager, such as the user's. */ +function adopted(proc: string, pid: number): boolean { + if (pid === 1) return true; + try { + return readFileSync(join(proc, String(pid), "comm"), "utf8").trim() === "systemd"; + } catch (error) { + if (["ENOENT", "ESRCH"].includes((error as NodeJS.ErrnoException).code ?? "")) return false; + throw error; + } +} + /** Whether `pid` is the harness `host` names, by its process ID or its name. */ function harnessIs(host: Host, pid: number): boolean { if ("pid" in host.harness) return host.harness.pid === pid; diff --git a/src/cli/program.ts b/src/cli/program.ts index e6d53d9e..138abb48 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -72,12 +72,15 @@ export function which(name: string, env: NodeJS.ProcessEnv): string | undefined * names a program many times. */ export function runs(env: NodeJS.ProcessEnv, cwd: string): (program: string, cd: string[]) => boolean { + const platform = process.platform; + const paths = platform === "win32" ? win32 : posix; const cache = new Map(); return (program, cd) => { - const path = program.includes("/") ? posix.resolve(cwd, ...cd, program) : program; + const named = platform === "win32" ? /[\\/]/.test(program) : program.includes("/"); + const path = named ? paths.resolve(cwd, ...cd, program) : program; let found = cache.get(path); if (found === undefined) { - found = program.includes("/") ? launchable(path, process.platform) : which(program, env) !== undefined; + found = named ? launchable(path, platform) : which(program, env) !== undefined; cache.set(path, found); } return found; diff --git a/src/core/command.ts b/src/core/command.ts index 6f50cbd9..00b8d05f 100644 --- a/src/core/command.ts +++ b/src/core/command.ts @@ -133,7 +133,7 @@ export function commandName(script: string, runs: Runs): string { /** The program an argument vector runs, as a record keeps it, reading a shell's `-c` script as `commandName` does. */ export function argvName(argv: string[], runs: Runs): string { const run = (program: string) => runs(program, []); - const program = argv[0]?.split("/").pop() ?? ""; + const program = argv[0]?.split(/[\\/]/).pop() ?? ""; if (["sh", "bash", "zsh", "dash"].includes(program) && argv[1] === "-c" && argv[2] !== undefined) { return commandName(argv[2], runs); } @@ -165,7 +165,7 @@ function simple(command: string[], runs: (program: string) => boolean): string | i++; continue; } - if (WRAPPERS.has(word.split("/").pop()!) && command[i + 1] !== undefined) { + if (WRAPPERS.has(word.split(/[\\/]/).pop()!) && command[i + 1] !== undefined) { i++; while (command[i] !== undefined && /^(?:-.*|\d+[smhd]?|[A-Za-z_][A-Za-z0-9_]*=.*)$/.test(command[i]!)) i++; continue; @@ -173,8 +173,13 @@ function simple(command: string[], runs: (program: string) => boolean): string | break; } const first = command[i]!; - const base = first.split("/").pop()!; - const program = PROGRAM.test(base) && ((!first.includes("/") && BUILTINS.has(first)) || runs(first)) ? base : "?"; + // A path's last part, by either separator, as Windows writes one, without a Windows program's extension. + const base = first + .split(/[\\/]/) + .pop()! + .replace(/\.(?:exe|cmd|bat|com)$/i, ""); + const path = /[\\/]/.test(first); + const program = PROGRAM.test(base) && ((!path && BUILTINS.has(first)) || runs(first)) ? base : "?"; const sub = SUBCOMMANDS.has(program) ? command[i + 1] : undefined; return sub !== undefined && VERBS.has(sub) ? `${program} ${sub}` : program; } diff --git a/test/calls.test.ts b/test/calls.test.ts index e88798e2..05dfabb1 100644 --- a/test/calls.test.ts +++ b/test/calls.test.ts @@ -59,6 +59,13 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { expect(sessionSample(at, { ...host(below, [10, 20, 30, 21], 30), harness: { pid: 10 } })).toEqual(none); }); + test("refuses a scope shared with another tree whose parent still runs elsewhere", () => { + // The app 2 starts the harness 10 and, in the same scope, an unrelated process 50. + const parents = { 2: 1, 10: 2, 20: 10, 30: 20, 50: 2 }; + const each = host(parents, [10, 20, 30, 50], 30); + expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); + }); + test("refuses a hook outside the cgroup it reads", () => { const each = host({ 10: 1, 30: 10 }, [10], 30); expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); diff --git a/test/command.test.ts b/test/command.test.ts index 8949fb01..8a44a79b 100644 --- a/test/command.test.ts +++ b/test/command.test.ts @@ -1,9 +1,11 @@ -import { posix } from "node:path"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join, posix } from "node:path"; import { expect, test } from "vitest"; import { COMMAND_LIMIT, argvName, commandName, isCommandName } from "../src/core"; import { runs } from "../src/cli/program"; +import { tempDir } from "./fixture"; /** The programs a host in these tests can run, by name or path. */ const installed = new Set( @@ -77,14 +79,30 @@ test.each([ "ls test", ])("%j is no command name", (name) => expect(isCommandName(name)).toBe(false)); -test("a host runs a name on PATH or a path to a launchable file, from its directory", () => { - const run = runs({ PATH: "/bin:/usr/bin" }, "/"); - const on = (program: string, cd: string[] = []) => run(program, cd); - expect(on("sh")).toBe(true); - expect(on("bin/sh")).toBe(true); - expect(on("hunter2")).toBe(false); - expect(on("./hunter2")).toBe(false); - expect(on("etc/passwd")).toBe(false); - expect(on("./sh", ["usr", "bin"])).toBe(true); - expect(on("./sh", ["usr"])).toBe(false); +// Windows finds a program by its extension rather than a mode bit. +test.skipIf(process.platform === "win32")( + "a host runs a name on PATH or a path to a launchable file, from its directory", + () => { + const root = tempDir(); + mkdirSync(join(root, "bin")); + writeFileSync(join(root, "bin", "tool"), "#!/bin/sh\n", { mode: 0o755 }); + writeFileSync(join(root, "bin", "data"), "x\n", { mode: 0o644 }); + const run = runs({ PATH: join(root, "bin") }, root); + const on = (program: string, cd: string[] = []) => run(program, cd); + expect(on("tool")).toBe(true); + expect(on("bin/tool")).toBe(true); + expect(on("hunter2")).toBe(false); + expect(on("./hunter2")).toBe(false); + expect(on("bin/data")).toBe(false); + expect(on("./tool", ["bin"])).toBe(true); + expect(on("./tool", ["."])).toBe(false); + }, +); + +test("a Windows path names its program by its last part, without its extension", () => { + const ran: string[] = []; + const host = (program: string) => (ran.push(program), true); + expect(argvName(["C:\\tools\\cargo.exe", "build"], host)).toBe("cargo build"); + expect(commandName("'C:\\tools\\gh.cmd' pr view 1", host)).toBe("gh pr"); + expect(ran).toEqual(["C:\\tools\\cargo.exe", "C:\\tools\\gh.cmd"]); }); From ace8dec5c667d8a3e1e6b24caec77d9e4e583236 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 11:14:11 -0800 Subject: [PATCH 10/13] fix(records): count an orphan tree only where its environment names the session Co-Authored-By: Claude Opus 5.5 --- src/cli/calls.ts | 31 ++++++++++++++++++++++++++----- src/cli/hooks.ts | 3 ++- test/calls.test.ts | 28 +++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/cli/calls.ts b/src/cli/calls.ts index 27e27f3b..1b7a76d5 100644 --- a/src/cli/calls.ts +++ b/src/cli/calls.ts @@ -127,6 +127,8 @@ export interface Host { * Codex's `codex`. */ harness: { pid: number } | { name: string }; + /** The hook's session ID, which the harness gives each command it runs in its environment. */ + session: string; } /** @@ -134,8 +136,8 @@ export interface Host { * Such a cgroup holds one tree of processes, whose top is the harness: the hook's topmost ancestor in the cgroup, from * which every other branch hangs, as when an app starts each agent session in a scope of its own. A terminal's scope * fails, since its top is the terminal's shell, as does one shared with other work. A process whose parent left, as - * a daemonized one's does, is the session's leftover and counts where init or a systemd manager adopted it; one - * whose parent runs outside the cgroup is other work. + * a daemonized one's does, is the session's leftover and counts where init or a systemd manager adopted it and its + * environment names the session, as the harness gives each command; any other is other work. */ export function sessionSample(now: number, host: Host): Sample { const at = new Date(now).toISOString(); @@ -166,9 +168,14 @@ export function sessionSample(now: number, host: Host): Sample { let at: number | undefined = parent(pid); while (at !== undefined && inside.has(at) && !chain.has(at)) at = parent(at); if (at !== undefined && chain.has(at) && at !== top) return none; - // A tree outside the harness's counts only as a daemon the session left, which init or the user's service - // manager, as the reaper of orphans, adopted. A live parent elsewhere means other work shares the cgroup. - if (at !== undefined && !chain.has(at) && !adopted(host.proc, at)) return none; + // A tree outside the harness's counts only as a daemon the session left: init or the user's service manager, + // as the reaper of orphans, adopted its root, whose environment names the session. A live parent elsewhere, or + // an orphan of another session's, means other work shares the cgroup. + if (at !== undefined && !chain.has(at)) { + let root = pid; + while (parent(root) !== at) root = parent(root)!; + if (!adopted(host.proc, at) || !ownedBy(host.proc, root, host.session)) return none; + } } const usage = cgroupUsage(group); return usage === null ? none : { at, cgroup: path, usage }; @@ -185,6 +192,20 @@ function adopted(proc: string, pid: number): boolean { } } +/** Whether `pid`'s environment names `session` as Claude Code's or Codex's, as each gives the commands it runs. */ +function ownedBy(proc: string, pid: number, session: string): boolean { + if (session === "") return false; + let environ: string; + try { + environ = readFileSync(join(proc, String(pid), "environ"), "utf8"); + } catch (error) { + if (["ENOENT", "ESRCH", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) return false; + throw error; + } + const names = environ.split("\0"); + return names.includes(`CLAUDE_CODE_SESSION_ID=${session}`) || names.includes(`CODEX_THREAD_ID=${session}`); +} + /** Whether `pid` is the harness `host` names, by its process ID or its name. */ function harnessIs(host: Host, pid: number): boolean { if ("pid" in host.harness) return host.harness.pid === pid; diff --git a/src/cli/hooks.ts b/src/cli/hooks.ts index f85c4cab..04db36ab 100644 --- a/src/cli/hooks.ts +++ b/src/cli/hooks.ts @@ -441,7 +441,8 @@ function sample(deps: HookDependencies, value: Partial): Sample { ? { pid } : undefined; if (!harness) return { at: new Date(now).toISOString(), cgroup: null, usage: null }; - return sessionSample(now, { proc: "/proc", cgroups: "/sys/fs/cgroup", self: process.pid, harness }); + const session = typeof value.session_id === "string" ? value.session_id : ""; + return sessionSample(now, { proc: "/proc", cgroups: "/sys/fs/cgroup", self: process.pid, harness, session }); } /** diff --git a/test/calls.test.ts b/test/calls.test.ts index 05dfabb1..60eabba9 100644 --- a/test/calls.test.ts +++ b/test/calls.test.ts @@ -17,7 +17,12 @@ import { tempDir } from "./fixture"; describe.skipIf(process.platform !== "linux")("sessionSample", () => { const scope = "/app.slice/agent.scope"; /** A fake /proc and cgroup tree: each process's parent, which are in the session's scope, and the name of 10. */ - function host(parents: Record, scoped: number[], self: number) { + function host( + parents: Record, + scoped: number[], + self: number, + sessions: Record = {}, + ) { const proc = tempDir(); const cgroups = tempDir(); for (const [pid, parent] of Object.entries(parents)) { @@ -25,12 +30,17 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { // The command name may hold spaces and parentheses. writeFileSync(join(proc, pid, "stat"), `${pid} (a (b) c) S ${parent} 1 1 0\n`); writeFileSync(join(proc, pid, "comm"), pid === "10" ? "codex\n" : "bash\n"); + const session = sessions[Number(pid)]; + writeFileSync( + join(proc, pid, "environ"), + session ? `PATH=/bin\0CLAUDE_CODE_SESSION_ID=${session}\0` : "PATH=/bin\0", + ); writeFileSync(join(proc, pid, "cgroup"), `0::${scoped.includes(Number(pid)) ? scope : "/other.scope"}\n`); } mkdirSync(join(cgroups, scope), { recursive: true }); writeFileSync(join(cgroups, scope, "cgroup.procs"), `${scoped.join("\n")}\n`); writeFileSync(join(cgroups, scope, "cpu.stat"), "usage_usec 7000\nuser_usec 5000\n"); - return { proc, cgroups, self }; + return { proc, cgroups, self, session: "s" }; } const at = Date.parse("2026-11-15T10:00:00Z"); const counted: Sample = { at: "2026-11-15T10:00:00.000Z", cgroup: scope, usage: 7000 }; @@ -40,7 +50,7 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { // The harness 10, its hook shell 20 and hook 30, an MCP server 11, a tool shell 12 and its child 13, and a // daemon 40 whose parent left. const parents = { 10: 1, 20: 10, 30: 20, 11: 10, 12: 10, 13: 12, 40: 1 }; - const each = host(parents, [10, 20, 30, 11, 12, 13, 40], 30); + const each = host(parents, [10, 20, 30, 11, 12, 13, 40], 30, { 40: "s" }); expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(counted); expect(sessionSample(at, { ...each, harness: { name: "codex" } })).toEqual(counted); // A harness that names itself, or runs under a name, elsewhere is not this scope's top. @@ -66,6 +76,18 @@ describe.skipIf(process.platform !== "linux")("sessionSample", () => { expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); }); + test("refuses an orphan whose environment names another session, or none", () => { + // Init adopted 50, as it would another harness whose launcher exited, or a daemon of another session's. + const parents = { 10: 1, 20: 10, 30: 20, 50: 1, 51: 50 }; + for (const sessions of [{}, { 50: "other" }] as Record[]) { + const each = host(parents, [10, 20, 30, 50, 51], 30, sessions); + expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); + } + // Its own daemon, with a child of its own, counts. + const own = host(parents, [10, 20, 30, 50, 51], 30, { 50: "s" }); + expect(sessionSample(at, { ...own, harness: { pid: 10 } })).toEqual(counted); + }); + test("refuses a hook outside the cgroup it reads", () => { const each = host({ 10: 1, 30: 10 }, [10], 30); expect(sessionSample(at, { ...each, harness: { pid: 10 } })).toEqual(none); From 1682969732f2e666ed4ed647b5be22a01bf1e802 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 00:54:54 -0800 Subject: [PATCH 11/13] fix(records): keep a gated command's exit status when its CPU cannot be read, and rank shared groups apart A failed cpu.stat read now warns and records cpu: null instead of failing a command that exited 0. The mistake audit ranks only unshared shell groups per command and lists shared ones apart. The README's example record now parses, and the footer-check paragraph names its subject. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- quest/m0/mistake-audit.md | 12 +++++++----- src/cli/supervise.ts | 7 +++++-- test/gate.test.ts | 26 +++++++++++++++++++++++++- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9c981345..db6ddf35 100644 --- a/README.md +++ b/README.md @@ -1770,7 +1770,7 @@ latest upload, since one sender sends a session's uploads in order. The type is `parseUpload`, which refuse a malformed record whole: ```json -{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null},{"kind":"gate-run","start":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","until":null}],"gated":[{"command":"just test","start":"2026-11-14T00:22:07.386Z","admitted":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","cpu":1214300}],"shell":[{"calls":[{"command":"git status --short","start":"2026-11-14T00:21:58.004Z","end":"2026-11-14T00:21:58.051Z"}],"cpu":30,"shared":false}],"split":null} +{"harness":"claude","session":"7ef44bda-fb9b-436f-9ed8-9013d0f38d46","parent":null,"repository":"acme/app","branch":"quest/m0/login","quest":"quest/m0/login.md","pulls":[42],"role":"implementing","models":[{"model":"claude-opus-5-5","effort":"high","input":230,"cacheWrite":211500,"cacheRead":15800000,"output":88800,"cost":6.64,"complete":true}],"cost":6.64,"complete":true,"start":"2026-11-14T00:06:12.270Z","end":"2026-11-14T00:31:40.120Z","waits":[{"kind":"gate-queue","start":"2026-11-14T00:22:07.386Z","end":"2026-11-14T00:22:07.402Z","until":null},{"kind":"gate-run","start":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","until":null}],"gated":[{"command":"just test","start":"2026-11-14T00:22:07.386Z","admitted":"2026-11-14T00:22:07.402Z","end":"2026-11-14T00:24:51.020Z","cpu":1214300}],"shell":[{"calls":[{"command":"git status","start":"2026-11-14T00:21:58.004Z","end":"2026-11-14T00:21:58.051Z"}],"cpu":30,"shared":false}],"split":null} ``` - `pulls` are the PRs it worked on, attributed as `quest usage` attributes @@ -2335,7 +2335,7 @@ Claude Code's and Codex's `Stop` hooks queue the session's also sample the CPU each shell call used, and Claude Code's `Stop` and `SubagentStop` which background commands still run. -It checks a body given by `--body`, a `body=` field, `git commit -m`, +The footer check reads a body given by `--body`, a `body=` field, `git commit -m`, `--body-file -` with a here-document, or a body file. A file is read only when its command is the only one on the command line; after another command, or piped into `gh`, a body passes unread, since it may hold diff --git a/quest/m0/mistake-audit.md b/quest/m0/mistake-audit.md index 5a564094..03ef4e58 100644 --- a/quest/m0/mistake-audit.md +++ b/quest/m0/mistake-audit.md @@ -52,15 +52,17 @@ Agent's choices, for the implementer to revisit. Each run: can still name a PR that left the work to a planned follow-up or kept older behavior. - Read both kinds of gating mistake from each record's `gated` and - `shell` lists: gated runs ranked by least CPU, and shell groups by most, - with no threshold, judging each command's class rather than one cheap - run of a usually heavy command. A `command` holds only program names and + `shell` lists: gated runs ranked by least CPU, and unshared shell groups + by most, with no threshold, judging each command's class rather than one + cheap run of a usually heavy command. List `shared` groups apart, as CPU + no command owns: their `cpu` also covers another session's calls in the + cgroup, such as a sub-agent's build, so ranking them per command would + make a cheap parent command look CPU-heavy. A `command` holds only program names and subcommands, such as `cd; cargo build`, so judge the class by those. Count `nix build` apart, since `nix-daemon` builds outside its gate's scope. A `shell` group's `cpu` is null outside a cgroup that holds the session alone, as in a terminal, so - say how many groups were unmeasured; a `shared` group's CPU also covers - another session's calls in its cgroup, such as a sub-agent's. Records only hold calls from hosts + say how many groups were unmeasured. Records only hold calls from hosts whose hooks `quest setup` updated since that PR. - `quest messages` returns every human turn as a `kind: "message"` entry, beside the `kind: "answer"` entries tallied below, so set aside machine-written diff --git a/src/cli/supervise.ts b/src/cli/supervise.ts index 8984946d..efe55065 100644 --- a/src/cli/supervise.ts +++ b/src/cli/supervise.ts @@ -73,6 +73,8 @@ export interface RunOptions { within?: string | null; /** The slice the scope runs in; the user manager's default, `app.slice`, when omitted. */ slice?: string; + /** Reads the CPU microseconds a measured run's cgroup used; `cgroupUsage` when omitted. */ + usage?(group: string): number | null; } export interface Scopes { @@ -479,9 +481,10 @@ function contain( let cpu: number | null = null; try { // Read before anything else, while the holder keeps the cgroup; a run cancelled hard may have lost it. - if (describe.measure && group) cpu = cgroupUsage(group); + if (describe.measure && group) cpu = (options.usage ?? cgroupUsage)(group); } catch (error) { - failure ??= error as Error; + // The reading only informs the record, so it never changes how the run ends. + process.stderr.write(`quest: cannot read the CPU ${describe.name} used: ${error}\n`); } hold?.destroy(); try { diff --git a/test/gate.test.ts b/test/gate.test.ts index 14998dab..7b9dcd32 100644 --- a/test/gate.test.ts +++ b/test/gate.test.ts @@ -43,7 +43,7 @@ import { } from "../src/cli/gate"; import { forwards, foregroundStat, statProcess } from "../src/cli/interrupt"; import { gateBoundary, main } from "../src/cli/main"; -import { alive as ownerAlive, questScope } from "../src/cli/supervise"; +import { measuredRun, alive as ownerAlive, questScope } from "../src/cli/supervise"; import { readWaits } from "../src/cli/waits"; import { created, onTerminal, run, signalled, tempDir } from "./fixture"; @@ -1098,6 +1098,30 @@ describe.runIf(process.platform === "linux")("a systemd scope", () => { await until(() => host.usage(unit) === undefined); }); + test("keeps a successful command's exit status when its CPU cannot be read", async () => { + const warn = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + try { + const result = await measuredRun( + `quest-gate-${randomUUID()}.scope`, + process.cwd(), + ["/bin/sh", "-c", "exit 0"], + { + cancel: cancellation().cancel, + within: null, + usage: () => { + throw Object.assign(new Error("EACCES: permission denied"), { code: "EACCES" }); + }, + }, + ); + expect(result).toEqual({ status: 0, cpu: null }); + expect(warn.mock.calls.map(([text]) => String(text)).join("")).toMatch( + /cannot read the CPU .* used: .*EACCES/, + ); + } finally { + warn.mockRestore(); + } + }); + test("reads a short command's CPU however soon systemd removes its scope", async () => { // systemd removes an empty scope's cgroup as its command exits, sometimes before the gate sees the exit. for (let i = 0; i < 20; i++) { From 77a1cd2b1f8505f1d52f38b59c84108fecfa7b38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 01:52:15 -0800 Subject: [PATCH 12/13] fix(records): drop links to the finished gating quest and stub the hook test's programs gate-record-cpu.md linked to and required quest/m0/benchmark/gating.md, which this PR deletes; it now names measuredRun instead. The hooks test's sampled() fixture put /bin:/usr/bin on PATH, where a NixOS runner has no cat or ls; it now builds its own stub programs. Co-Authored-By: Claude Opus 5.5 --- quest/m0/benchmark/gate-record-cpu.md | 7 +------ test/hooks.test.ts | 7 +++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/quest/m0/benchmark/gate-record-cpu.md b/quest/m0/benchmark/gate-record-cpu.md index 620e30a8..8e8db620 100644 --- a/quest/m0/benchmark/gate-record-cpu.md +++ b/quest/m0/benchmark/gate-record-cpu.md @@ -17,7 +17,7 @@ stays in the journal; and Windows, which still refuses the variable. Decided 2026-11-15 (maintainer, `/quest-complete` of [kixel-ai/kixel.quest#56](https://github.com/kixel-ai/kixel.quest/pull/56), then `/quest-plan`): ✅ Linux, reusing the measurement -[gating](/quest/m0/benchmark/gating.md) adds (rejected: also a +`measuredRun` in `src/cli/supervise.ts` makes (rejected: also a reaped-children lower bound on process-group gates, which means rewriting the group owner for a macOS host that has not arrived; no quest, with the job page reading the `gated` list from uploaded session records, which @@ -40,8 +40,3 @@ Agent's choices, for the implementer to revisit: - Keep `unit`, which still names the scope's journal line for peak memory. - Update the README's "Record each run" section to say where CPU and wall time come from. - -## Required - -- [Gating](/quest/m0/benchmark/gating.md) - measures each gated run's CPU - and end time, which this copies into the record diff --git a/test/hooks.test.ts b/test/hooks.test.ts index ac7e5324..0e3d712b 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -221,9 +221,12 @@ test("SessionStart adds nothing where the repository does not use Quest", () => expect(hook({ hook_event_name: "SessionStart", cwd: tempDir() })).toBe(""); }); -/** Hook dependencies writing to a fresh state directory, sampling a fixed counter. */ +/** Hook dependencies writing to a fresh state directory, sampling a fixed counter, with `cat` and `ls` on PATH. */ function sampled(usage = 5_000) { - const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir(), PATH: "/bin:/usr/bin" }; + // Stub programs, since a host such as NixOS keeps neither in /bin or /usr/bin. + const bin = tempDir(); + for (const program of ["cat", "ls"]) writeFileSync(join(bin, program), "#!/bin/sh\n", { mode: 0o755 }); + const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir(), PATH: bin }; const sample = () => ({ at: "2026-11-15T10:00:00.000Z", cgroup: "/app.slice/agent.scope", usage }); return { env, sample, lines: () => readCalls(Date.parse("2026-11-15T00:00:00Z"), env) }; } From 73aefc3bfa1068ac6e3aa1d7a101971dbf04ee20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 01:56:13 -0800 Subject: [PATCH 13/13] test(hooks): stub the fixture's programs as .cmd on Windows Co-Authored-By: Claude Opus 5.5 --- test/hooks.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/hooks.test.ts b/test/hooks.test.ts index 0e3d712b..0fac6126 100644 --- a/test/hooks.test.ts +++ b/test/hooks.test.ts @@ -223,9 +223,10 @@ test("SessionStart adds nothing where the repository does not use Quest", () => /** Hook dependencies writing to a fresh state directory, sampling a fixed counter, with `cat` and `ls` on PATH. */ function sampled(usage = 5_000) { - // Stub programs, since a host such as NixOS keeps neither in /bin or /usr/bin. + // Stub programs, since a host such as NixOS keeps neither in /bin or /usr/bin; Windows launches by extension. const bin = tempDir(); - for (const program of ["cat", "ls"]) writeFileSync(join(bin, program), "#!/bin/sh\n", { mode: 0o755 }); + const stub = process.platform === "win32" ? ".cmd" : ""; + for (const program of ["cat", "ls"]) writeFileSync(join(bin, program + stub), "#!/bin/sh\n", { mode: 0o755 }); const env = { XDG_STATE_HOME: tempDir(), XDG_CONFIG_HOME: tempDir(), PATH: bin }; const sample = () => ({ at: "2026-11-15T10:00:00.000Z", cgroup: "/app.slice/agent.scope", usage }); return { env, sample, lines: () => readCalls(Date.parse("2026-11-15T00:00:00Z"), env) };