From ce029bc580d9082d35617d96d6c86154242a98fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 22:25:32 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m0/agent-admission.md Quest-Claim: 584031747fd1d419 From 09503bd924c2af69474f60cb3be790668d5a1600 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 23:13:16 -0800 Subject: [PATCH 2/6] feat(agent): admit an agent only while the host and account can carry it Co-Authored-By: Claude Fable 5.2 --- README.md | 26 ++++ assets/AGENTS.md | 4 +- assets/CONTEXT.md | 2 +- assets/skills/run.md | 2 +- docs/context.md | 2 +- quest/m0/agent-admission-measure.md | 33 ++++ quest/m0/agent-admission.md | 96 ------------ quest/m0/agent-usage.md | 48 ++++++ quest/m0/gate-memory.md | 4 - src/cli/admission.ts | 232 ++++++++++++++++++++++++++++ src/cli/agent.ts | 49 +++++- src/cli/gate.ts | 51 +++++- src/cli/main.ts | 39 ++++- test/admission.test.ts | 188 ++++++++++++++++++++++ test/agent.test.ts | 46 +++++- 15 files changed, 707 insertions(+), 115 deletions(-) create mode 100644 quest/m0/agent-admission-measure.md delete mode 100644 quest/m0/agent-admission.md create mode 100644 quest/m0/agent-usage.md create mode 100644 src/cli/admission.ts create mode 100644 test/admission.test.ts diff --git a/README.md b/README.md index 4d445923..24881bbc 100644 --- a/README.md +++ b/README.md @@ -1663,6 +1663,7 @@ quest agent start --harness claude --model claude-opus-5-5 --effort high --permi quest agent resume --harness claude --model claude-opus-5-5 --effort high --permission-mode auto .worktrees/foo "Take the recommendation." quest agent stop .worktrees/foo quest agent stop --all +quest agent admit --harness claude ``` Codex starts and resumes only in a checkout registered by `quest worktree create`, @@ -1676,6 +1677,31 @@ pruned dependencies when the next turn needs them. Claude continues to run directly in the supplied checkout. Codex desktop threads use the desktop app's native archival instead. +Start and resume first wait until the host and the account can carry one more +agent, before claiming the worktree, and a coordinator runs `quest agent admit +--harness ` before it starts or resumes a sub-agent of its own: + +- The host has room while `quest gate` would admit a command at once, counting + every command queued for it and, as one recent admission each, every agent + admitted in the last 5 minutes: an agent adds no gated load until its first + build, and three quarters of those that build at all reach it by then. An + agent `start` or `resume` runs stops counting when it exits. As in the gate, + a host loaded only by work no gate holds still starts one agent at a time. +- The account has room while, in each usage window its harness reports, the + usage spent plus what this agent and the harness's agents in flight on this + host are expected to spend stays within the window. Codex reports its windows + in its session logs; an agent's expected share is the token-weighted mean + share of this host's sessions since the window opened, so a host with no + history there expects none. Claude Code reports its windows only to an + interactive session's status line, so its agents are admitted on the host + alone, and each admission says so. + +Admission only delays the next agent: none already running is paused or +stopped, and `quest agent stop` does not see a launch still waiting, which ends +when its command is cancelled. `admit` exits 0 once admitted, and all three exit +3 at once, naming the reset, when the account cannot cover the agent; start and +resume then print `outcome limited` and no session. + Options go before the worktree, and the prompt or message follows it, so text starting with `-` is never read as an option. The agent runs in the worktree with the user's environment minus the coordinating session's harness variables diff --git a/assets/AGENTS.md b/assets/AGENTS.md index a5a2b322..2cd6566c 100644 --- a/assets/AGENTS.md +++ b/assets/AGENTS.md @@ -192,7 +192,9 @@ Relay no interim report: resume a sub-agent that ended its turn on one, telling A long-running coordinator lets compaction bound its context: what must outlast it lives in the forge and the quest tree, and a PR's state is read afresh before acting, not trusted from earlier turns. Start each sub-agent in the background with its own isolated worktree, since agents cannot write to a worktree their parent created. -It reports only when done or blocked, and never waits on GitHub CI, reviews, or merges; stay silent on its interim notifications. +Before starting or resuming one, run `quest agent admit --harness ` in the background and go on once it exits 0, handling reports meanwhile: it waits until the host has room for another agent. `quest agent start` and `resume` admit themselves. +Its exit 3 means the account's reported usage cannot cover another agent until the reset it names: a usage limit, handled as below. Report once a harness it says reports no usage. +A sub-agent reports only when done or blocked, and never waits on GitHub CI, reviews, or merges; stay silent on its interim notifications. As each reports, explain its result in a few lines: its PR, whether it is still a draft, and its open decisions with recommendations. Once two agents report the same blocker, such as a check failure that reproduces on the base or a reviewer outage, name it with its evidence in each later agent's brief, and report it once. diff --git a/assets/CONTEXT.md b/assets/CONTEXT.md index ec3e8439..3ae8ccf2 100644 --- a/assets/CONTEXT.md +++ b/assets/CONTEXT.md @@ -44,7 +44,7 @@ Before ending a task, pausing for a decision, or handing off, commit a checkpoin Finish with a clean checkout: intended changes committed and pushed, incidental generated changes restored, and pre-existing or unrelated changes preserved. Run CPU-heavy commands, such as builds, tests, model checkers, and browsers, through `quest gate `, which queues until the host has room; model calls like `quest review` and servers stay out. -Never work around the queue by polling the load average or capping agents yourself; cancel a gated command only when you no longer need its result. +Never work around the queue by polling the load average or capping agents yourself, since `quest agent admit` decides when one starts; cancel a gated command only when you no longer need its result. Run it in the foreground and read its exit code directly, not through a pipe. Never end a turn or report while your background work runs, and never poll it with `sleep` or status checks: block on it with the harness's own wait for that task, at the longest interval it allows, saying nothing between waits. To wait on a PR's CI or reviews, use `quest wait`, re-running one that times out with the `--since` it printed, so activity during the first wait is still reported. diff --git a/assets/skills/run.md b/assets/skills/run.md index 942f969c..573d4588 100644 --- a/assets/skills/run.md +++ b/assets/skills/run.md @@ -16,7 +16,7 @@ A brief gives the task, its reason and context, such as the quest or PR and the Starting quests asks the user nothing: each agent works headless, and its decisions surface in its PR. -`quest unblocked --unclaimed --json` ranks the unclaimed ready quests, reading the upstream tree afresh; start them in that order, as many at once as the harness allows. +`quest unblocked --unclaimed --json` ranks the unclaimed ready quests, reading the upstream tree afresh; start them in that order, each once `quest agent admit` admits it, as the guide's Coordination section describes. It lists each skipped claim on stderr, with how long ago it was last active when it has an open PR; name any that look stale in the summary, for the user to take over. Start every quest you would recommend: well planned, with no blockers, and no conflict with work already started. For one waiting on an outside condition, first check whether it has cleared. diff --git a/docs/context.md b/docs/context.md index 37285247..1f914323 100644 --- a/docs/context.md +++ b/docs/context.md @@ -44,7 +44,7 @@ Before ending a task, pausing for a decision, or handing off, commit a checkpoin Finish with a clean checkout: intended changes committed and pushed, incidental generated changes restored, and pre-existing or unrelated changes preserved. Run CPU-heavy commands, such as builds, tests, model checkers, and browsers, through `quest gate `, which queues until the host has room; model calls like `quest review` and servers stay out. -Never work around the queue by polling the load average or capping agents yourself; cancel a gated command only when you no longer need its result. +Never work around the queue by polling the load average or capping agents yourself, since `quest agent admit` decides when one starts; cancel a gated command only when you no longer need its result. Run it in the foreground and read its exit code directly, not through a pipe. Never end a turn or report while your background work runs, and never poll it with `sleep` or status checks: block on it with the harness's own wait for that task, at the longest interval it allows, saying nothing between waits. To wait on a PR's CI or reviews, use `quest wait`, re-running one that times out with the `--since` it printed, so activity during the first wait is still reported. diff --git a/quest/m0/agent-admission-measure.md b/quest/m0/agent-admission-measure.md new file mode 100644 index 00000000..a035d335 --- /dev/null +++ b/quest/m0/agent-admission-measure.md @@ -0,0 +1,33 @@ +# [S] Agent admission is measured on a real fan-out + +## Goal + +A `/quest-spawn` fan-out on the maintainer's host, run with agent admission, +is compared with the three weeks before it: usage-limit errors, agents lost +to them, gate waits over 10 and 30 minutes, and total wall time. The numbers +go in this quest's PR, with any change to admission they call for. + +## Plan + +Baseline, from transcripts on the maintainer's three Linux hosts, 2026-10-22 +to 11-12: fan-outs of 20 to 26 sub-agents; 157 "All credentials ... cooling +down" errors on one host; 156 background agents "terminated early due to an +API error"; and of 2,482 gated commands on one host, 191 waited over 10 +minutes, 77 over 30, and 11 were still queued at the 55-minute timeout. + +[moq-dev/quest#692](https://github.com/moq-dev/quest/pull/692) could not run +one: every host's load was above its CPU count, with gated commands queued, +so admission held every agent, which is its job and no fan-out. + +Agent's choices, for the implementer to revisit: + +- Give each agent its own `QUEST_GATE_RECORD` file, and read the waits from + those rather than from transcripts. +- Check that the 5 minutes an admitted agent counts against the host + (`PENDING_MS` in `src/cli/admission.ts`) still covers three quarters of the + agents that gate anything, and that waves of admissions do not arrive at + the gate together. + +## Required + +- [The gate records each run for the process that started the agent](/quest/m0/gate-record.md) - each agent's gate waits, read without matching transcripts diff --git a/quest/m0/agent-admission.md b/quest/m0/agent-admission.md deleted file mode 100644 index f1e3bcd8..00000000 --- a/quest/m0/agent-admission.md +++ /dev/null @@ -1,96 +0,0 @@ -# [L] Coordinators start an agent only when the host and the account can carry it - -## Goal - -Every skill that starts agents, such as `/quest-run`, `/quest-spawn`, -`/quest-complete`, and `/quest-audit`, and `quest agent start` and -`quest agent resume`, starts or resumes an agent only while the host's gate -would admit work at once and the account's remaining usage covers the agents -already in flight. A fan-out, including the resumes after a usage limit -resets, never dies en masse on a usage limit, and a started agent rarely -stalls in the gate with its prompt cache going cold. Agents already running -are never paused or stopped by this; it only delays starting the next one. - -Non-goals: capping the user's own top-level sessions, querying a -particular credential gateway, and changing how the gate admits commands. - -## Plan - -Found 2026-11-12 from transcripts on the maintainer's three Linux hosts, -2026-10-22 to 11-12: - -- Fan-outs peaked at 20 to 26 concurrent sub-agents per session (Claude - Code caps native sub-agents at 20 per session, "Concurrent subagent limit - reached"), and up to 59 top-level sessions were active on one host within - five minutes. Nothing caps headless `quest agent start` agents or Codex. -- The shared quota is the main harm: 157 "All credentials ... cooling - down" errors on one host, 143 of them on the day of the 59-session peak; - weekly and session limits hit on several days; 156 background agents - ended "terminated early due to an API error", 26 of them within seconds - in one moq session. -- Gate waits: of 2,482 gated commands on one host, 191 waited over 10 - minutes, 77 over 30, and 11 were still queued at the 55-minute command - timeout. A waiting agent's turn blocks, so its cache can expire. -- Agent processes themselves are cheap: about 100 to 250 MB and 1% CPU - each. - -Decided 2026-11-12 (maintainer, /quest-plan): - -- What outcome? ✅ Admission on both quota and host (rejected: quota only, - which leaves the gate waits; host only, which lets several hosts drain one - account; a fixed smaller fan-out in skill text, which is blind to quota - and load). -- What says the host can carry one more agent? ✅ The gate would admit a - command at once (rejected: a fixed agent count per host, a guess; a - count derived from cores and memory, a poor proxy for gated load). It - targets the measured harm and adds no constant. -- What says the account can? ✅ The usage each harness reports, such as - Codex's `rate_limits.used_percent` in its session log and Claude's - 5-hour and weekly utilization where its login or gateway exposes it. - Stop starting once the budget left cannot cover what the agents in flight - are expected to spend, estimated from `quest usage`'s per-size history - (rejected: asking the credential gateway's management API, which ties the - core to one gateway against AGENTS.md; starting in small waves with no - signal). Decide at a coordinator's start which setups give a signal, and - say once when one gives none, rather than trying and falling back. -- Milestone: ✅ m0 (rejected: m1), since both harms are live. - -What the decisions above require, found in review: - -- "Would admit at once" is the gate's own admission decision (load, - memory, and the headroom recent admissions take), not an empty queue: a - host can be full with nothing queued, and an agent just started adds no - gated load until its first build minutes later. So agents started but - not yet building must count against the headroom too, as the gate counts - its recent admissions, or a whole fan-out passes the check at once. -- Harnesses report a percentage of a window, while `quest usage` reports - tokens and list cost, and neither reports the window's capacity. Convert - through the account's own history, such as tokens spent per percentage - point over recent sessions, or an absolute budget where a harness exposes - one. - -Agent's choices, for the implementer to revisit: - -- Admission waits in the coordinator before it starts or resumes an agent, - never inside a started agent, so only one cheap context waits. `quest - agent start` and `quest agent resume` can admit themselves; native - sub-agents need a step the coordinator runs first, since Claude Code's - `SubagentStart` hook fires once the sub-agent exists and runs `quest - hook` by default (`quest worktree hook` only with `quest setup - --worktrees`). Keep the coordinator able to handle reports while it - waits. -- Verify whether Claude's utilization reaches a session through the - maintainer's credential-pool gateway (`ANTHROPIC_BASE_URL` on a local - port); if not, that setup reports no signal rather than a wrong one. -- `assets/CONTEXT.md` tells agents "never work around the queue by - polling the load average or capping agents yourself". Keep the ban on - agents improvising caps, and point to the admission instead. The - "as many at once as the harness allows" line in `assets/skills/run.md` - changes with it. -- Measure on a real `/quest-spawn` fan-out: limit errors, agents lost, - gate wait times, and total wall time, against the numbers above. - -## Related - -- [The gate records each run for the process that started the agent](/quest/m0/gate-record.md) - per-agent gate waits, to measure this against -- [Cut agent token waste](/quest/m0/token-waste/README.md) - kept the 1-hour cache TTL and capped commands just under it, which a long gate wait can still outlast diff --git a/quest/m0/agent-usage.md b/quest/m0/agent-usage.md new file mode 100644 index 00000000..0f674d87 --- /dev/null +++ b/quest/m0/agent-usage.md @@ -0,0 +1,48 @@ +# [M] Agent admission reads Claude's usage and sizes an agent's share + +## Goal + +`quest agent admit`, `start`, and `resume` refuse a Claude agent its account's +usage cannot cover, as they already do for Codex, and expect more of the +account from a large quest's agent than from a small one's. + +## Plan + +Found 2026-11-12 while building agent admission +([moq-dev/quest#692](https://github.com/moq-dev/quest/pull/692)), which +checks the host for every harness and the account for Codex alone: + +- Claude Code 2.1.322 gives no command its usage windows. Its docs name them + only in the JSON a `statusLine` command receives (`rate_limits.five_hour` + and `rate_limits.seven_day`, each with `used_percentage` and `resets_at`), + for claude.ai subscribers. A headless `claude -p --output-format + stream-json --verbose` turn on the CLI's own login printed no + `rate_limit_event`, which the SDK docs say comes only when a limit's status + changes, and no transcript or file under `~/.claude` holds the windows. So + admission says Claude reports no usage and checks the host alone. +- Sessions through a credential-pool gateway (`ANTHROPIC_BASE_URL`) have no + single account: a pool's windows are its credentials', which only the + gateway knows, and asking it is a non-goal of admission. +- An agent's expected share is the token-weighted mean share of the host's + sessions since the window opened, whatever its quest's size. `quest + usage`'s per-size history, which the plan named, took 21 seconds and a + listing of the repository's PRs per call on the maintainer's host, and + attributed 29 of 964 sessions to a PR, with no row for L or XL. + +Agent's choices, for the implementer to revisit: + +- Read Claude's windows from a status line: `quest setup` installs a + `statusLine` command that records the `rate_limits` it is given, chaining + to the one the user already has, and admission reads the newest record. + It covers only hosts where an interactive session runs on the account. +- For sizes, take `--size` on `admit`, `start`, and `resume`, and scale the + host's mean share by the size's share of spend where `quest usage` has + history for it, read from a cache a coordinator refreshes once per run. + +## Waiting + +- Maintainer: whether an agent's expected share stays the host's mean for every size (recommended: the per-size history is too thin to change a decision, and a wrong size estimate refuses agents the account could carry) or takes the quest's size as above. An agent cannot pick: the plan decided per-size history before the data showed how little of it there is. Check: this item is answered in the Plan. + +## Related + +- [Cut agent token waste](/quest/m0/token-waste/README.md) - the other half of what an account's usage buys diff --git a/quest/m0/gate-memory.md b/quest/m0/gate-memory.md index d349bced..41da5227 100644 --- a/quest/m0/gate-memory.md +++ b/quest/m0/gate-memory.md @@ -56,7 +56,3 @@ Agent's choices, for the implementer to revisit: ## Hosts - linux - systemd scopes carry the memory limits - -## Related - -- [Coordinators start an agent only when the host and the account can carry it](/quest/m0/agent-admission.md) - the agent-level half of host admission diff --git a/src/cli/admission.ts b/src/cli/admission.ts new file mode 100644 index 00000000..4054a623 --- /dev/null +++ b/src/cli/admission.ts @@ -0,0 +1,232 @@ +// Agent admission: a coordinator starts or resumes an agent only while the host's gate would admit +// work at once and the account's reported usage covers the agents in flight. It only delays the next +// start: agents already running are never paused or stopped. +import { randomUUID } from "node:crypto"; +import { closeSync, fstatSync, openSync, readSync, readdirSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, join } from "node:path"; + +import { + type Admission, + type Host, + type Limits, + SETTLE_MS, + type Waiting, + claimDirectory, + fileHost, + hostLimits, + processStart, + room, +} from "./gate"; + +/** + * How long an admitted agent counts against the host's headroom, as one recent admission of the + * gate's: an agent adds no gated load until its first build. Of 263 sub-agents on one host over three + * weeks, the 126 that gated anything reached their first gated command in a median of 133 seconds, + * and three quarters within 265; the rest never gated, so no later event ends the count. + */ +export const PENDING_MS = 5 * 60_000; + +/** One usage window an account's harness reports. */ +export interface Window { + /** The percentage of the window spent. */ + used: number; + /** When the window resets, in epoch milliseconds. */ + resets: number; + /** The percentage one more agent is expected to spend, from this host's sessions in the window. */ + expected: number; +} + +/** What `admit` reads and writes, replaceable in tests. */ +export interface AdmissionDependencies { + limits: Limits; + /** The gate's ledger and the host's load and memory. */ + gate: Host; + /** The agents' ledger; only its clock, ledger, liveness, and sleep are used. */ + agents: Host; + /** The usage windows `harness` reports for its account, or null when it reports none. */ + account(harness: string): Window[] | null; + /** A line for the user. */ + notice(text: string): void; +} + +/** The account cannot cover one more agent until `resets`, in epoch milliseconds. */ +export interface Refusal { + resets: number; +} + +const waits: Record = { + load: "the load average to fall", + memory: "free memory", + queue: "gated commands queued on this host to start", +}; + +/** + * Wait until the host can carry one more agent of `harness`, then record it. The host has room while + * the gate would admit a command at once (see `room`), counting each agent admitted in the last + * PENDING_MS whose launcher, if it has one, still runs. The account has room while, in every window + * its harness reports, the usage spent plus what this agent and the harness's agents in flight are + * expected to spend stays within the window; in flight are the agents whose launcher still runs, and + * those without one admitted in the last PENDING_MS. Returns null once admitted, or why the account + * cannot cover it, which no wait here clears. A harness that reports no usage is admitted on the + * host alone, saying so. + * + * `held` is the process that runs the agent, as `quest agent start` does; without it, as for a + * coordinator's own sub-agent, nothing reports the agent's end. + */ +export async function admit( + harness: string, + deps: AdmissionDependencies, + signal: AbortSignal, + held?: number, +): Promise { + // Decided once, at the start: a setup either reports its usage or does not. + const windows = deps.account(harness); + if (windows === null) deps.notice(`${harness} reports no account usage here, so only the host is checked`); + const holder = held === undefined ? {} : { pid: held, start: processStart(held) }; + const shown = new Set(); + for (;;) { + signal.throwIfAborted(); + const ledger = deps.agents.ledger(); + const now = deps.agents.now(); + const kept = ledger.admissions + .filter((entry) => (entry.pid === undefined ? entry.at > now - PENDING_MS : deps.agents.running(entry))) + // A clock stepped backward dates an entry in the future; count it from now instead. + .map((entry) => (entry.at > now ? { ...entry, at: now } : entry)); + const flying = kept.filter((entry) => entry.agent === harness).length; + const short = (windows ?? []) + .filter((window) => window.resets > now && window.used + (flying + 1) * window.expected > 100) + .map((window) => window.resets); + if (short.length > 0) return { resets: Math.max(...short) }; + const pending = kept.filter((entry) => entry.at > now - PENDING_MS).length; + const reason = room(deps.limits, deps.gate, pending); + if (reason === null) { + const entry: Admission = { id: randomUUID(), at: now, ...holder, agent: harness }; + if (deps.agents.publish(ledger, [...kept, entry])) return null; + continue; + } + if (!shown.has(reason)) { + shown.add(reason); + deps.notice(`waiting for ${waits[reason]} before starting an agent`); + } + await deps.agents.sleep(SETTLE_MS, signal); + } +} + +/** How much of a rollout's end is read for its last usage report: a report is about a kilobyte. */ +const TAIL = 256 * 1024; + +/** The last `token_count` event in the last TAIL bytes of the Codex rollout at `path`, if any. */ +function lastCount(path: string): { tokens: number; limits: Record | null } | undefined { + const fd = openSync(path, "r"); + let text: string; + try { + const size = fstatSync(fd).size; + const buffer = Buffer.alloc(Math.min(size, TAIL)); + readSync(fd, buffer, 0, buffer.length, size - buffer.length); + text = buffer.toString("utf8"); + } finally { + closeSync(fd); + } + const lines = text.split("\n"); + for (let index = lines.length - 1; index >= 0; index--) { + if (!lines[index].includes('"token_count"')) continue; + let payload: Record | undefined; + try { + payload = (JSON.parse(lines[index]) as { payload?: Record }).payload; + } catch { + continue; // The tail's first line is cut, and a session still writing leaves its last one open. + } + if (payload?.type !== "token_count") continue; + const total = (payload.info as { total_token_usage?: { total_tokens?: unknown } } | null)?.total_token_usage; + const limits = payload.rate_limits; + return { + tokens: typeof total?.total_tokens === "number" ? total.total_tokens : 0, + limits: typeof limits === "object" && limits !== null ? (limits as Record) : null, + }; + } + return undefined; +} + +/** + * The usage windows Codex last reported for its account, from the `rate_limits` of the newest + * session rollout in `home` that holds one, or null when none does. A window's capacity in tokens is + * not reported, so an agent's expected share comes from the account's own history: the sessions this + * host wrote since the window opened spent `used` between them, and an agent is expected to spend + * the share of the session a token most likely belongs to, their token-weighted mean, since most + * spend comes from the long sessions an agent's is. Other hosts' spend on the account raises `used` + * without adding sessions here, which only makes the estimate more careful. + */ +export function codexAccount(home: string, now: number): Window[] | null { + const rollouts: { path: string; modified: number }[] = []; + const walk = (dir: string) => { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const entry of entries) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.name.endsWith(".jsonl")) rollouts.push({ path, modified: statSync(path).mtimeMs }); + } + }; + walk(join(home, "sessions")); + rollouts.sort((a, b) => b.modified - a.modified); + const counts = new Map>(); + const count = (path: string) => { + if (!counts.has(path)) counts.set(path, lastCount(path)); + return counts.get(path); + }; + let reported: Record | undefined; + for (const { path } of rollouts) { + reported = count(path)?.limits ?? undefined; + if (reported !== undefined) break; + } + if (reported === undefined) return null; + const windows: Window[] = []; + for (const name of ["primary", "secondary"]) { + const window = reported[name] as { + used_percent?: unknown; + window_minutes?: unknown; + resets_at?: unknown; + } | null; + if (window == null) continue; + const { used_percent: used, window_minutes: minutes, resets_at: resets } = window; + if (typeof used !== "number" || typeof minutes !== "number" || typeof resets !== "number") + throw new Error(`Codex reported a ${name} usage window quest cannot read: ${JSON.stringify(window)}`); + // A window that has reset since the report holds usage nothing here has read yet. + if (resets * 1000 <= now) continue; + const opened = resets * 1000 - minutes * 60_000; + const spent = rollouts.filter(({ modified }) => modified >= opened).map(({ path }) => count(path)?.tokens ?? 0); + const total = spent.reduce((sum, tokens) => sum + tokens, 0); + const squares = spent.reduce((sum, tokens) => sum + tokens * tokens, 0); + windows.push({ used, resets: resets * 1000, expected: total > 0 ? (used * squares) / (total * total) : 0 }); + } + return windows; +} + +/** The agents' ledger of this host, beside the gate's. */ +const agentsDirectory = (gate: string) => join(dirname(dirname(gate)), "agents", basename(gate)); + +/** + * The admission `quest agent` runs with: the host's own limits and ledgers, and each harness's own + * report. Claude Code reports its usage windows only to an interactive session's status line, and a + * headless turn prints none, so it has no signal to read. + */ +export function processAdmission(notice: (text: string) => void, env = process.env): AdmissionDependencies { + const gate = claimDirectory(); + return { + limits: hostLimits(), + gate: fileHost(gate), + agents: fileHost(agentsDirectory(gate)), + account: (harness) => + harness === "codex" ? codexAccount(env.CODEX_HOME || join(homedir(), ".codex"), Date.now()) : null, + notice, + }; +} + +/** The admission every `quest agent` command runs, held in an object so a test can replace it. */ +export const admission = { admit }; diff --git a/src/cli/agent.ts b/src/cli/agent.ts index 63c4e1a5..4b9dab4a 100644 --- a/src/cli/agent.ts +++ b/src/cli/agent.ts @@ -24,7 +24,9 @@ import { constants, tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; +import { type Refusal, admission, processAdmission } from "./admission"; import { discoveryEnv, git } from "./git"; +import { cancelled, interrupts } from "./interrupt"; import { LIMIT } from "./limit"; import { alive, programCommand, programEnvironment, requireProgram, signalProcess } from "./program"; import { systemdScopes } from "./supervise"; @@ -386,14 +388,52 @@ export interface AgentDependencies { base: string; stdout(text: string): void; stderr(text: string): void; + /** + * Wait until the host can carry one more agent of `harness`, run by process `held` if any, and + * return null, or why its account cannot cover one (see admission.ts). + */ + admit(harness: HarnessName, signal: AbortSignal, held?: number): Promise; } export const processDependencies = (out: Pick): AgentDependencies => ({ env: process.env, base: agentDirectory(), ...out, + admit: (harness, signal, held) => + admission.admit( + harness, + processAdmission((text) => out.stderr(`quest: ${text}\n`)), + signal, + held, + ), }); +/** + * Admit one agent of `harness` before it starts, for `quest agent admit` and every launch. Returns + * undefined once admitted, else the exit status: the usage limit's when the account cannot cover + * the agent, or the signal's when interrupted while waiting. + */ +export async function admitAgent( + harness: HarnessName, + deps: AgentDependencies, + held?: number, +): Promise { + const cancel = interrupts(); + try { + const refusal = await deps.admit(harness, cancel.signal, held); + if (refusal === null) return undefined; + deps.stderr( + `quest: ${harness}'s reported usage cannot cover another agent; resets ${new Date(refusal.resets).toISOString()}\n`, + ); + return EXIT.limited; + } catch (error) { + if (cancel.signal.aborted) return cancelled(cancel.signal); + throw error; + } finally { + cancel.close(); + } +} + /** How much of an agent's output is kept for diagnostics; the rest is read as it streams. */ const TAIL = 64 * 1024; const tail = (value: string) => (value.length > TAIL ? value.slice(-TAIL) : value); @@ -432,7 +472,8 @@ export function boundedLines(limit: number, read: (line: string) => void) { } /** - * Run one agent turn in the foreground: a new session with `prompt`, or + * Run one agent turn in the foreground, once the host and account can carry it + * (see `admitAgent`): a new session with `prompt`, or * `session` resumed with it. Prints the session id as soon as it is known, * then the outcome and final reply. Returns the exit status. */ @@ -467,6 +508,12 @@ export async function runAgent( const program = requireProgram(name, deps.env); const command = checkouts ? undefined : programCommand(program, args, deps.env); const dir = spawnDirectory(deps.base, worktree); + // Waited out before the worktree is claimed, so a queued launch holds nothing. + const refused = await admitAgent(launch.harness, deps, process.pid); + if (refused !== undefined) { + if (refused === EXIT.limited) deps.stdout("outcome limited\n"); + return refused; + } const reader = harness.reader(); // Claude Code takes the id this command assigns; Codex announces its own. diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 2fdaa6d6..172261ee 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -27,6 +27,8 @@ export interface Admission { queued?: boolean; start?: string; cores?: number; + /** In the agents' ledger (see admission.ts), the harness of the agent this entry admitted. */ + agent?: string; } /** One version of the host's admission ledger and queue. Versions only grow; 0 is the empty ledger. */ @@ -257,7 +259,8 @@ function parse(text: string): Admission[] { (entry.pid === undefined || (Number.isSafeInteger(entry.pid) && entry.pid > 0)) && (entry.queued === undefined || entry.queued === true) && (entry.start === undefined || typeof entry.start === "string") && - (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)), + (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)) && + (entry.agent === undefined || typeof entry.agent === "string"), ); return valid ? (value as Admission[]) : []; } @@ -413,6 +416,44 @@ export function fileHost(directory: string, now = Date.now): Host { }; } +/** + * The load and memory a caller is admitted against while the `running` admissions hold the host: + * the load counts as at least the CPUs they kept busy, and without one the raw load counts as + * having room (see `admit`). A priority caller counts no load. + */ +function standing(limits: Limits, host: Host, running: Admission[], priority = false) { + const used = priority ? 0 : running.reduce((sum, admission) => sum + (admission.cores ?? 0), 0); + const load = priority ? 0 : running.length > 0 ? Math.max(host.load(), used) : Math.min(host.load(), limits.load); + return { used, load, memory: host.memory() }; +} + +/** + * Why a command arriving now would wait, or null when the gate would admit it at once even with + * `reserved` more recent admissions counted against the headroom: work admitted elsewhere that has + * not reached the gate yet, such as an agent still reading (see admission.ts). It counts every + * queued caller as ahead, and never the spaced admissions `admit` grants on raw room, which would + * pass the reserved work one by one. Like the gate, it admits a lone arrival whatever the load: + * with nothing gated running, recent, queued, or reserved, the load is work no gate can wait out. + */ +export function room(limits: Limits, host: Host, reserved: number): Waiting | null { + const ledger = host.ledger(); + const now = host.now(); + const live = ledger.admissions.filter((admission) => host.running(admission)); + const queued = live.filter( + (admission) => admission.queued && (admission.start !== undefined || admission.at > now - RECENT_MS), + ).length; + const admissions = ledger.admissions.filter((admission) => !admission.queued); + const running = admissions.filter((admission) => live.includes(admission)); + const recent = admissions.filter((admission) => admission.at > now - RECENT_MS).length; + const { used, load, memory } = standing(limits, host, running); + if (load > limits.load || used >= limits.load) return "load"; + if (memory < limits.memory) return "memory"; + const taken = recent + queued + reserved; + if (running.length === 0 && taken === 0) return null; + if (load + taken >= limits.load) return queued > 0 ? "queue" : "load"; + return memory - taken * limits.memory >= limits.memory ? null : queued > 0 ? "queue" : "memory"; +} + /** * Wait until the host has room for a command, then record its admission. The * load average and available memory lag new work, so each admission from the @@ -510,13 +551,7 @@ export async function admit( // Recorded like any admission, so it still counts against the others' headroom. if (now >= deadline) return admitted(now); // Priority ignores load, whether or not a gated command runs; memory still applies. - const used = precedence.priority ? 0 : [...running].reduce((sum, admission) => sum + (admission.cores ?? 0), 0); - const load = precedence.priority - ? 0 - : running.size > 0 - ? Math.max(host.load(), used) - : Math.min(host.load(), limits.load); - const memory = host.memory(); + const { used, load, memory } = standing(limits, host, [...running], precedence.priority); let wait = SETTLE_MS; if (load > limits.load || used >= limits.load) waiting?.("load"); else if (memory < limits.memory) waiting?.("memory"); diff --git a/src/cli/main.ts b/src/cli/main.ts index 1cd95c8b..a8639f1a 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -23,8 +23,19 @@ import { onMaintainer, waiting, } from "../core"; -import { AgentInputError, HARNESSES, type Launch, processDependencies, runAgent, stopAgent, stopAll } from "./agent"; +import { + AgentInputError, + HARNESSES, + type HarnessName, + type Launch, + admitAgent, + processDependencies, + runAgent, + stopAgent, + stopAll, +} from "./agent"; import { held, openDrafts, reach, target, upstream, upstreamRefs } from "./claims"; +import { PENDING_MS } from "./admission"; import { gate, QUICK_MS } from "./gate"; import { gitHubDependencies, @@ -1022,6 +1033,8 @@ export function main(argv: string[], out: Output): number | Promise { "The agent runs in with the user's environment minus every CLAUDE* (except CLAUDE_CONFIG_DIR), " + "ANTHROPIC*, CODEX_* (except CODEX_HOME), OP_*, OPENAI_API_KEY, QUEST_GATE_ADMITTED, QUEST_GATE_PRIORITY, " + "QUEST_LAND_BASE, and MBX_SCHEDULER_PRIORITY when QUEST_GATE_PRIORITY=1, so it uses the CLI's stored login rather than the coordinating session's.\n\n" + + "Start and resume first wait until the host and account can carry the agent, as agent admit describes, " + + "before claiming the worktree, and exit 3 with `outcome limited` and no session when the account cannot.\n\n" + "Prints `session ` as soon as the id is known, then `outcome finished|failed|limited|logged-out|stopped`, " + "a blank line, and the agent's final reply. Exits 0 when the agent finished, 3 when its account hit a " + "usage limit or ran out of credit, 4 when the CLI is logged out (after printing its login command), 2 for " + @@ -1070,6 +1083,30 @@ export function main(argv: string[], out: Output): number | Promise { ).action((worktree: string, session: string, message: string, options: LaunchOptions) => launch(worktree, message, session, options), ); + agent + .command("admit") + .summary("Wait until the host and account can carry one more agent") + .description( + "Wait until this host can carry one more agent, for a coordinator to run before it starts or resumes " + + "an agent of its own harness, such as a sub-agent; start and resume do the same themselves. The " + + "host has room while quest gate would admit a command at once, counting every queued command and, " + + `as one recent admission each, every agent admitted in the last ${PENDING_MS / 60_000} minutes, since an agent ` + + "adds no gated load until its first build; an agent start or resume runs counts only until it exits. Agents " + + "already running are never paused.\n\n" + + "The account has room while, in each usage window its harness reports, the usage spent plus what this " + + "agent and the harness's agents in flight on this host are expected to spend stays within the window. " + + "Codex reports its windows in its session logs, read from $CODEX_HOME or ~/.codex, and an agent's " + + "expected share is the token-weighted mean share of this host's sessions since the window opened. " + + "Claude Code reports its windows to no command, so it is admitted on the host alone, saying so.\n\n" + + "Exits 0 once admitted, 3 at once when the account cannot cover the agent, naming the reset, 128 plus " + + "the signal number when interrupted, and 1 otherwise.", + ) + .addOption( + new Option("--harness ", "Agent CLI the agent runs on").choices(HARNESSES).makeOptionMandatory(), + ) + .action((options: { harness: HarnessName }) => { + pending = admitAgent(options.harness, agentDependencies()).then((status) => status ?? 0, agentFailure); + }); agent .command("stop") .summary("Stop the agent in a worktree, or end the spawn with --all") diff --git a/test/admission.test.ts b/test/admission.test.ts new file mode 100644 index 00000000..d1ad769a --- /dev/null +++ b/test/admission.test.ts @@ -0,0 +1,188 @@ +import { mkdirSync, utimesSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expect, test } from "vitest"; +import { type AdmissionDependencies, PENDING_MS, type Window, admit, codexAccount } from "../src/cli/admission"; +import { type Admission, type Host, RECENT_MS, SETTLE_MS, fileHost, processStart } from "../src/cli/gate"; +import { tempDir } from "./fixture"; + +const GiB = 2 ** 30; +const signal = new AbortController().signal; +const me = { pid: process.pid, start: processStart(process.pid) }; + +/** A host of 8 CPUs with real ledgers, and load, memory, a clock, and process liveness the test controls. */ +function host(windows: Window[] | null = null) { + const state = { now: 10 * PENDING_MS, load: 0, memory: Infinity, dead: new Set(), stop: Infinity }; + const sleeps: number[] = []; + const notices: string[] = []; + const ledger = (directory: string): Host => ({ + ...fileHost(directory, () => state.now), + load: () => state.load, + memory: () => state.memory, + running: (admission) => admission.pid !== undefined && !state.dead.has(admission.pid), + sleep: async (ms) => { + // A wait that outlasts the test's patience ends it, as a cancelled caller's would. + if (state.now >= state.stop) throw new Error("still waiting"); + sleeps.push(ms); + state.now += ms; + }, + }); + const deps: AdmissionDependencies = { + limits: { load: 8, memory: 4 * GiB }, + gate: ledger(tempDir()), + agents: ledger(tempDir()), + account: () => windows, + notice: (text) => notices.push(text), + }; + const gated = (admissions: Admission[]) => deps.gate.publish(deps.gate.ledger(), admissions); + /** Whether one more agent is admitted without waiting. */ + const atOnce = async (harness = "claude", held?: number) => { + state.stop = state.now; + try { + return (await admit(harness, deps, signal, held)) === null; + } catch (error) { + if ((error as Error).message !== "still waiting") throw error; + return false; + } + }; + return { state, sleeps, notices, deps, gated, atOnce }; +} + +test("a fan-out fills the host's headroom, and each agent frees its share once it would have built", async () => { + const { state, deps, sleeps, notices, atOnce } = host(); + state.load = 5; + // Load 5 plus 0, 1, and 2 agents not yet building stays under 8 CPUs; a fourth would not. + expect([await atOnce(), await atOnce(), await atOnce(), await atOnce()]).toEqual([true, true, true, false]); + // The fourth waits as the gate's callers do, and starts once the first three no longer count. + state.stop = Infinity; + expect(await admit("claude", deps, signal)).toBeNull(); + expect(sleeps.reduce((sum, ms) => sum + ms, 0)).toBe(PENDING_MS); + expect(new Set(sleeps)).toEqual(new Set([SETTLE_MS])); + // Each waiting call names what it waits for once, however long it waits. + expect(notices.filter((text) => text.startsWith("waiting"))).toEqual( + Array(2).fill("waiting for the load average to fall before starting an agent"), + ); +}); + +test("each agent not yet building takes one command's memory", async () => { + const { state, atOnce } = host(); + state.memory = 9 * GiB; + expect([await atOnce(), await atOnce(), await atOnce()]).toEqual([true, true, false]); +}); + +test("a queued or running gated command holds agents back as it would another command", async () => { + const { state, gated, atOnce, notices } = host(); + state.load = 6; + // One command queued for the gate: an arrival would wait behind it only once the host is short. + gated([{ id: "queued", at: state.now, ...me, queued: true }]); + expect([await atOnce(), await atOnce()]).toEqual([true, false]); + expect(notices.at(-1)).toContain("gated commands queued"); + // Running commands that kept every CPU busy hold the host whatever the load average says. + state.load = 0; + state.now += PENDING_MS; + gated([{ id: "build", at: state.now - RECENT_MS, ...me, cores: 8 }]); + expect(await atOnce()).toBe(false); + // A recent gated admission counts once, as the gate counts it. + gated([{ id: "build", at: state.now, ...me, cores: 1 }]); + state.load = 6; + expect([await atOnce(), await atOnce()]).toEqual([true, false]); +}); + +test("a host loaded by work no gate holds still starts one agent at a time", async () => { + const { state, atOnce } = host(); + state.load = 40; + expect([await atOnce(), await atOnce()]).toEqual([true, false]); + state.now += PENDING_MS; + expect(await atOnce()).toBe(true); + // Memory is never waived. + state.now += PENDING_MS; + state.memory = GiB; + expect(await atOnce()).toBe(false); +}); + +test("an agent whose launcher exited no longer counts, and one still running counts only while it may yet build", async () => { + const { state, atOnce } = host(); + state.load = 7; + expect([await atOnce("codex", 4242), await atOnce("codex")]).toEqual([true, false]); + state.dead.add(4242); + expect(await atOnce("codex")).toBe(true); + state.now += PENDING_MS; + expect([await atOnce("codex", 4243), await atOnce("codex")]).toEqual([true, false]); + state.now += PENDING_MS; + expect(await atOnce("codex")).toBe(true); +}); + +test("the account admits an agent only while its windows cover it and the agents in flight", async () => { + const resets = 20 * PENDING_MS; + const { state, deps, atOnce, notices } = host([ + { used: 70, resets, expected: 10 }, + { used: 99, resets: 5 * PENDING_MS, expected: 10 }, + ]); + // The second window reset before now, so it holds nothing; 70 + 10 per agent covers three. + expect([await atOnce("codex", 1), await atOnce("codex", 2), await atOnce("codex", 3)]).toEqual([true, true, true]); + expect(await admit("codex", deps, signal)).toEqual({ resets }); + // Another harness's agents spend another account. + expect(await atOnce("claude")).toBe(true); + // An agent that ended is no longer in flight, however recently it started. + state.dead.add(2); + expect(await atOnce("codex")).toBe(true); + // One without a launcher counts as in flight only while it counts against the host. + expect(await admit("codex", deps, signal)).toEqual({ resets }); + state.now += PENDING_MS; + expect(await atOnce("codex")).toBe(true); + expect(notices).toEqual([]); +}); + +test("a harness that reports no usage is admitted on the host alone, saying so", async () => { + const { atOnce, notices } = host(null); + expect(await atOnce("claude")).toBe(true); + expect(notices).toEqual(["claude reports no account usage here, so only the host is checked"]); +}); + +/** A Codex rollout in `home` last written at `modified` ms, ending with a `token_count` event. */ +function rollout(home: string, name: string, modified: number, tokens: number, limits: unknown, tail = "") { + const dir = join(home, "sessions", "2026", "11", "12"); + mkdirSync(dir, { recursive: true }); + const path = join(dir, `rollout-${name}.jsonl`); + const event = { + type: "event_msg", + payload: { type: "token_count", info: { total_token_usage: { total_tokens: tokens } }, rate_limits: limits }, + }; + writeFileSync(path, `{"type":"session_meta"}\n${JSON.stringify(event)}\n${tail}`); + utimesSync(path, modified / 1000, modified / 1000); +} + +test("Codex's windows come from its newest report, with an agent's share from this host's sessions", () => { + const home = tempDir(); + const minute = 60_000; + const now = 1_000_000 * minute; + const week = { used_percent: 40, window_minutes: 10_080, resets_at: (now + 1000 * minute) / 1000 }; + const hours = { used_percent: 10, window_minutes: 300, resets_at: (now + 100 * minute) / 1000 }; + const old = { primary: { ...week, used_percent: 5 }, secondary: null }; + // Written before the 5-hour window opened, inside the week's. + rollout(home, "a", now - 400 * minute, 300, old); + rollout(home, "b", now - 50 * minute, 100, old); + // The newest rollout's last line is still being written, so its last whole report counts. + rollout( + home, + "c", + now - minute, + 100, + { primary: week, secondary: hours }, + '{"type":"event_msg","payload":{"type":"token_c', + ); + const [weekly, hourly] = codexAccount(home, now)!; + // Sessions of 300, 100, and 100 tokens: a token's session holds (9 + 1 + 1) / 25 of the window's spend. + expect(weekly).toEqual({ used: 40, resets: now + 1000 * minute, expected: (40 * 11) / 25 }); + expect(hourly).toEqual({ used: 10, resets: now + 100 * minute, expected: 10 / 2 }); + // A window that reset since the report is unknown, not spent. + expect(codexAccount(home, now + 200 * minute)).toEqual([expect.objectContaining({ used: 40 })]); +}); + +test("Codex without a usage report gives no signal, and an unreadable one fails", () => { + const home = tempDir(); + expect(codexAccount(home, 0)).toBeNull(); + rollout(home, "a", 1000, 100, null); + expect(codexAccount(home, 0)).toBeNull(); + rollout(home, "b", 2000, 100, { primary: { used_percent: "40%" }, secondary: null }); + expect(() => codexAccount(home, 0)).toThrow("Codex reported a primary usage window quest cannot read"); +}); diff --git a/test/agent.test.ts b/test/agent.test.ts index c175554a..1af56b47 100644 --- a/test/agent.test.ts +++ b/test/agent.test.ts @@ -12,8 +12,9 @@ import { writeFileSync, } from "node:fs"; import { delimiter, join } from "node:path"; -import { afterEach, expect, test, vi } from "vitest"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { admission } from "../src/cli/admission"; import { boundedLines, spawnDirectory, strippedEnvironment } from "../src/cli/agent"; import { batchArgument, programCommand, programEnvironment, signalProcess, which } from "../src/cli/program"; import { type Output, main } from "../src/cli/main"; @@ -21,6 +22,11 @@ import { tempDir } from "./fixture"; import { systemdScopes } from "../src/cli/supervise"; import { Worktrees } from "../src/cli/worktree"; +// Launches here are admitted at once, whatever the host running the tests carries. +beforeEach(() => { + vi.spyOn(admission, "admit").mockResolvedValue(null); +}); + afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); @@ -365,6 +371,44 @@ test("an account at its usage limit exits 3, and the session resumes once it res expect(call.stdin).toBe("continue"); }); +test("a launch waits for admission before it claims the worktree, and an account that cannot cover it exits 3", async () => { + const s = setup(); + const admit = vi.spyOn(admission, "admit").mockImplementation(async (harness, deps, signal, held) => { + expect([harness, held, signal.aborted]).toEqual(["claude", process.pid, false]); + // Nothing holds the worktree while the launch waits. + const state = join(s.runtime, "quest", "agent"); + expect(readdirSync(join(state, readdirSync(state)[0], "agents"))).toEqual([]); + deps.notice("waiting for the load average to fall before starting an agent"); + return null; + }); + const started = await quest("agent", "start", ...launch("claude"), s.worktree, "go"); + expect(started).toMatchObject({ + code: 0, + stderr: "quest: waiting for the load average to fall before starting an agent\n", + }); + + admit.mockResolvedValue({ resets: Date.UTC(2026, 10, 14, 15) }); + const session = /^session (\S+)$/m.exec(started.stdout)![1]; + const resumed = await quest("agent", "resume", ...launch("claude"), s.worktree, session, "continue"); + const refusal = "quest: claude's reported usage cannot cover another agent; resets 2026-11-14T15:00:00.000Z\n"; + expect(resumed).toEqual({ code: 3, stdout: "outcome limited\n", stderr: refusal }); + expect(s.calls()).toHaveLength(1); +}); + +test("agent admit exits 0 once admitted and 3 when the account cannot cover an agent of its own", async () => { + const admit = vi.spyOn(admission, "admit").mockResolvedValue(null); + expect(await quest("agent", "admit", "--harness", "codex")).toEqual({ code: 0, stdout: "", stderr: "" }); + // No process runs a coordinator's own sub-agent, so nothing holds its admission. + expect(admit.mock.calls[0][0]).toBe("codex"); + expect(admit.mock.calls[0][3]).toBeUndefined(); + admit.mockResolvedValue({ resets: 0 }); + expect(await quest("agent", "admit", "--harness", "codex")).toMatchObject({ + code: 3, + stderr: "quest: codex's reported usage cannot cover another agent; resets 1970-01-01T00:00:00.000Z\n", + }); + expect((await quest("agent", "admit")).code).toBe(2); +}); + test("a spent free usage allowance exits 3 as limited", async () => { const s = setup(); vi.stubEnv("FAKE_MODE", "free-limited"); From 99554a1a675708a9855c5e44885a7f7fd696ea41 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 23:49:48 -0800 Subject: [PATCH 3/6] context: fit the admission pointer within the hook's context limit Co-Authored-By: Claude Fable 5.2 --- assets/CONTEXT.md | 2 +- docs/context.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/CONTEXT.md b/assets/CONTEXT.md index 3ae8ccf2..77d6033a 100644 --- a/assets/CONTEXT.md +++ b/assets/CONTEXT.md @@ -44,7 +44,7 @@ Before ending a task, pausing for a decision, or handing off, commit a checkpoin Finish with a clean checkout: intended changes committed and pushed, incidental generated changes restored, and pre-existing or unrelated changes preserved. Run CPU-heavy commands, such as builds, tests, model checkers, and browsers, through `quest gate `, which queues until the host has room; model calls like `quest review` and servers stay out. -Never work around the queue by polling the load average or capping agents yourself, since `quest agent admit` decides when one starts; cancel a gated command only when you no longer need its result. +Never work around the queue by polling the load or capping agents, `quest agent admit`'s job; cancel a gated command only once its result is unneeded. Run it in the foreground and read its exit code directly, not through a pipe. Never end a turn or report while your background work runs, and never poll it with `sleep` or status checks: block on it with the harness's own wait for that task, at the longest interval it allows, saying nothing between waits. To wait on a PR's CI or reviews, use `quest wait`, re-running one that times out with the `--since` it printed, so activity during the first wait is still reported. diff --git a/docs/context.md b/docs/context.md index 1f914323..6324350c 100644 --- a/docs/context.md +++ b/docs/context.md @@ -44,7 +44,7 @@ Before ending a task, pausing for a decision, or handing off, commit a checkpoin Finish with a clean checkout: intended changes committed and pushed, incidental generated changes restored, and pre-existing or unrelated changes preserved. Run CPU-heavy commands, such as builds, tests, model checkers, and browsers, through `quest gate `, which queues until the host has room; model calls like `quest review` and servers stay out. -Never work around the queue by polling the load average or capping agents yourself, since `quest agent admit` decides when one starts; cancel a gated command only when you no longer need its result. +Never work around the queue by polling the load or capping agents, `quest agent admit`'s job; cancel a gated command only once its result is unneeded. Run it in the foreground and read its exit code directly, not through a pipe. Never end a turn or report while your background work runs, and never poll it with `sleep` or status checks: block on it with the harness's own wait for that task, at the longest interval it allows, saying nothing between waits. To wait on a PR's CI or reviews, use `quest wait`, re-running one that times out with the `--since` it printed, so activity during the first wait is still reported. From 0c0147eb5f7e670fe6941c10c4c2ab1af71a149f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 00:14:49 -0800 Subject: [PATCH 4/6] fix(agent): read Codex usage afresh, by event time and account, counting only a window's tokens Co-Authored-By: Claude Fable 5.2 --- README.md | 4 +- src/cli/admission.ts | 201 ++++++++++++++++++++++++++++++----------- src/cli/gate.ts | 2 +- src/cli/lines.ts | 27 ++++++ src/cli/main.ts | 2 +- src/cli/messages.ts | 3 +- src/cli/usage.ts | 28 +----- test/admission.test.ts | 138 +++++++++++++++++++++------- 8 files changed, 289 insertions(+), 116 deletions(-) create mode 100644 src/cli/lines.ts diff --git a/README.md b/README.md index 24881bbc..5008dc82 100644 --- a/README.md +++ b/README.md @@ -1688,9 +1688,9 @@ agent, before claiming the worktree, and a coordinator runs `quest agent admit agent `start` or `resume` runs stops counting when it exits. As in the gate, a host loaded only by work no gate holds still starts one agent at a time. - The account has room while, in each usage window its harness reports, the - usage spent plus what this agent and the harness's agents in flight on this + usage spent plus what this agent and the account's agents in flight on this host are expected to spend stays within the window. Codex reports its windows - in its session logs; an agent's expected share is the token-weighted mean + in its session logs, read afresh at each decision; an agent's expected share is the token-weighted mean share of this host's sessions since the window opened, so a host with no history there expects none. Claude Code reports its windows only to an interactive session's status line, so its agents are admitted on the host diff --git a/src/cli/admission.ts b/src/cli/admission.ts index 4054a623..475edc53 100644 --- a/src/cli/admission.ts +++ b/src/cli/admission.ts @@ -2,7 +2,7 @@ // work at once and the account's reported usage covers the agents in flight. It only delays the next // start: agents already running are never paused or stopped. import { randomUUID } from "node:crypto"; -import { closeSync, fstatSync, openSync, readSync, readdirSync, statSync } from "node:fs"; +import { closeSync, fstatSync, openSync, readFileSync, readSync, readdirSync, realpathSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; @@ -18,6 +18,7 @@ import { processStart, room, } from "./gate"; +import { eachLine } from "./lines"; /** * How long an admitted agent counts against the host's headroom, as one recent admission of the @@ -37,6 +38,13 @@ export interface Window { expected: number; } +/** An account's usage as its harness last reported it. */ +export interface Account { + /** Names the account among every harness's, so its agents in flight count against it alone. */ + id: string; + windows: Window[]; +} + /** What `admit` reads and writes, replaceable in tests. */ export interface AdmissionDependencies { limits: Limits; @@ -44,8 +52,8 @@ export interface AdmissionDependencies { gate: Host; /** The agents' ledger; only its clock, ledger, liveness, and sleep are used. */ agents: Host; - /** The usage windows `harness` reports for its account, or null when it reports none. */ - account(harness: string): Window[] | null; + /** The account `harness` runs on with its usage as of now, or null when the harness reports none. */ + account(harness: string): Account | null; /** A line for the user. */ notice(text: string): void; } @@ -65,11 +73,12 @@ const waits: Record = { * Wait until the host can carry one more agent of `harness`, then record it. The host has room while * the gate would admit a command at once (see `room`), counting each agent admitted in the last * PENDING_MS whose launcher, if it has one, still runs. The account has room while, in every window - * its harness reports, the usage spent plus what this agent and the harness's agents in flight are + * its harness reports, the usage spent plus what this agent and the account's agents in flight are * expected to spend stays within the window; in flight are the agents whose launcher still runs, and - * those without one admitted in the last PENDING_MS. Returns null once admitted, or why the account - * cannot cover it, which no wait here clears. A harness that reports no usage is admitted on the - * host alone, saying so. + * those without one admitted in the last PENDING_MS. The usage is read afresh at every decision, + * since agents spend it while this one waits for the host. Returns null once admitted, or why the + * account cannot cover it, which no wait here clears. A harness that reports no usage is admitted + * on the host alone, saying so. * * `held` is the process that runs the agent, as `quest agent start` does; without it, as for a * coordinator's own sub-agent, nothing reports the agent's end. @@ -81,27 +90,30 @@ export async function admit( held?: number, ): Promise { // Decided once, at the start: a setup either reports its usage or does not. - const windows = deps.account(harness); - if (windows === null) deps.notice(`${harness} reports no account usage here, so only the host is checked`); + const first = deps.account(harness); + if (first === null) deps.notice(`${harness} reports no account usage here, so only the host is checked`); const holder = held === undefined ? {} : { pid: held, start: processStart(held) }; const shown = new Set(); - for (;;) { + for (let account = first; ; account = first && deps.account(harness)) { signal.throwIfAborted(); + if (first !== null && account?.id !== first.id) + throw new Error(`${harness} stopped reporting the usage of the account this admission began on`); const ledger = deps.agents.ledger(); const now = deps.agents.now(); const kept = ledger.admissions .filter((entry) => (entry.pid === undefined ? entry.at > now - PENDING_MS : deps.agents.running(entry))) // A clock stepped backward dates an entry in the future; count it from now instead. .map((entry) => (entry.at > now ? { ...entry, at: now } : entry)); - const flying = kept.filter((entry) => entry.agent === harness).length; - const short = (windows ?? []) + const id = account?.id ?? harness; + const flying = kept.filter((entry) => entry.agent === id).length; + const short = (account?.windows ?? []) .filter((window) => window.resets > now && window.used + (flying + 1) * window.expected > 100) .map((window) => window.resets); if (short.length > 0) return { resets: Math.max(...short) }; const pending = kept.filter((entry) => entry.at > now - PENDING_MS).length; const reason = room(deps.limits, deps.gate, pending); if (reason === null) { - const entry: Admission = { id: randomUUID(), at: now, ...holder, agent: harness }; + const entry: Admission = { id: randomUUID(), at: now, ...holder, agent: id }; if (deps.agents.publish(ledger, [...kept, entry])) return null; continue; } @@ -116,8 +128,35 @@ export async function admit( /** How much of a rollout's end is read for its last usage report: a report is about a kilobyte. */ const TAIL = 256 * 1024; +/** One `token_count` event of a Codex rollout: when, the session's tokens so far, and the account's windows. */ +interface Count { + at: number; + tokens: number; + limits: Record | null; +} + +/** The `token_count` event on a rollout's `line`, if it is one and whole. */ +function count(line: string): Count | undefined { + if (!line.includes('"token_count"')) return undefined; + let event: { timestamp?: unknown; payload?: Record }; + try { + event = JSON.parse(line) as typeof event; + } catch { + return undefined; // A tail's first line is cut, and a session still writing leaves its last one open. + } + const at = typeof event.timestamp === "string" ? Date.parse(event.timestamp) : NaN; + if (event.payload?.type !== "token_count" || Number.isNaN(at)) return undefined; + const total = (event.payload.info as { total_token_usage?: { total_tokens?: unknown } } | null)?.total_token_usage; + const limits = event.payload.rate_limits; + return { + at, + tokens: typeof total?.total_tokens === "number" ? total.total_tokens : 0, + limits: typeof limits === "object" && limits !== null ? (limits as Record) : null, + }; +} + /** The last `token_count` event in the last TAIL bytes of the Codex rollout at `path`, if any. */ -function lastCount(path: string): { tokens: number; limits: Record | null } | undefined { +function lastCount(path: string): Count | undefined { const fd = openSync(path, "r"); let text: string; try { @@ -130,35 +169,49 @@ function lastCount(path: string): { tokens: number; limits: Record= 0; index--) { - if (!lines[index].includes('"token_count"')) continue; - let payload: Record | undefined; - try { - payload = (JSON.parse(lines[index]) as { payload?: Record }).payload; - } catch { - continue; // The tail's first line is cut, and a session still writing leaves its last one open. - } - if (payload?.type !== "token_count") continue; - const total = (payload.info as { total_token_usage?: { total_tokens?: unknown } } | null)?.total_token_usage; - const limits = payload.rate_limits; - return { - tokens: typeof total?.total_tokens === "number" ? total.total_tokens : 0, - limits: typeof limits === "object" && limits !== null ? (limits as Record) : null, - }; + const found = count(lines[index]); + if (found) return found; } return undefined; } +/** The tokens the session in the rollout at `path` had spent by `time`, from its last report before then. */ +function tokensBefore(path: string, time: number): number { + let tokens = 0; + eachLine(path, (line) => { + const found = count(line); + if (found === undefined) return; + if (found.at >= time) return false; + tokens = found.tokens; + }); + return tokens; +} + /** - * The usage windows Codex last reported for its account, from the `rate_limits` of the newest - * session rollout in `home` that holds one, or null when none does. A window's capacity in tokens is - * not reported, so an agent's expected share comes from the account's own history: the sessions this - * host wrote since the window opened spent `used` between them, and an agent is expected to spend - * the share of the session a token most likely belongs to, their token-weighted mean, since most - * spend comes from the long sessions an agent's is. Other hosts' spend on the account raises `used` - * without adding sessions here, which only makes the estimate more careful. + * What earlier reads of each rollout found, by path, so an admission that waits reads only what + * changed: its last report as of `modified`, and its tokens before each window's opening. */ -export function codexAccount(home: string, now: number): Window[] | null { - const rollouts: { path: string; modified: number }[] = []; +export type RolloutCache = Map }>; + +/** When a rollout's session started, from the local time its file name carries, or 0 when it has none. */ +function started(path: string): number { + const [, year, month, day, hour, minute, second] = + /rollout-(\d{4})-(\d\d)-(\d\d)T(\d\d)-(\d\d)-(\d\d)-/.exec(basename(path)) ?? []; + return year === undefined ? 0 : new Date(+year, +month - 1, +day, +hour, +minute, +second).getTime(); +} + +/** + * The usage windows Codex last reported for the account logged in at `home`, from the newest + * `rate_limits` event among its session rollouts, archived ones included, or null when none holds + * one. A window's capacity in tokens is not reported, so an agent's expected share comes from the + * account's own history: the sessions this host wrote since the window opened spent `used` between + * them, counting only their tokens since then, and an agent is expected to spend the share of the + * session a token most likely belongs to, their token-weighted mean, since most spend comes from the + * long sessions an agent's is. Other hosts' spend on the account raises `used` without adding + * sessions here, which only makes the estimate more careful. + */ +export function codexAccount(home: string, now: number, cache: RolloutCache = new Map()): Account | null { + const found = new Map(); const walk = (dir: string) => { let entries; try { @@ -170,25 +223,44 @@ export function codexAccount(home: string, now: number): Window[] | null { for (const entry of entries) { const path = join(dir, entry.name); if (entry.isDirectory()) walk(path); - else if (entry.name.endsWith(".jsonl")) rollouts.push({ path, modified: statSync(path).mtimeMs }); + // Archiving moves a rollout, so a scan racing it may see both copies of one session. + else if (entry.name.endsWith(".jsonl") && !found.has(entry.name)) + found.set(entry.name, { path, modified: statSync(path).mtimeMs }); } }; walk(join(home, "sessions")); - rollouts.sort((a, b) => b.modified - a.modified); - const counts = new Map>(); - const count = (path: string) => { - if (!counts.has(path)) counts.set(path, lastCount(path)); - return counts.get(path); + walk(join(home, "archived_sessions")); + const rollouts = [...found.values()].sort((a, b) => b.modified - a.modified); + const modifiedAt = new Map(rollouts.map(({ path, modified }) => [path, modified])); + const read = (path: string) => { + const modified = modifiedAt.get(path)!; + const known = cache.get(path); + if (known?.modified === modified) return known; + const entry = { modified, last: lastCount(path), before: known?.before ?? new Map() }; + cache.set(path, entry); + return entry; + }; + const lastOf = (path: string) => read(path).last; + // What a session had spent by a past time never changes, however much it writes later. + const before = (path: string, time: number) => { + const known = cache.get(path)?.before.get(time); + if (known !== undefined) return known; + const tokens = tokensBefore(path, time); + read(path).before.set(time, tokens); + return tokens; }; - let reported: Record | undefined; - for (const { path } of rollouts) { - reported = count(path)?.limits ?? undefined; - if (reported !== undefined) break; + // A rollout written after its last report can be newer than one holding a later report, but no + // report is newer than its file, so the search ends at the first file older than the best report. + let reported: Count | undefined; + for (const { path, modified } of rollouts) { + if (reported !== undefined && modified < reported.at) break; + const each = lastOf(path); + if (each?.limits && (reported === undefined || each.at > reported.at)) reported = each; } - if (reported === undefined) return null; + if (!reported?.limits) return null; const windows: Window[] = []; for (const name of ["primary", "secondary"]) { - const window = reported[name] as { + const window = reported.limits[name] as { used_percent?: unknown; window_minutes?: unknown; resets_at?: unknown; @@ -200,12 +272,34 @@ export function codexAccount(home: string, now: number): Window[] | null { // A window that has reset since the report holds usage nothing here has read yet. if (resets * 1000 <= now) continue; const opened = resets * 1000 - minutes * 60_000; - const spent = rollouts.filter(({ modified }) => modified >= opened).map(({ path }) => count(path)?.tokens ?? 0); + const spent = rollouts + .filter(({ modified }) => modified >= opened) + .map(({ path }) => { + const total = lastOf(path)?.tokens ?? 0; + // A session resumed across the reset spent part of its tokens in an earlier window. + return total > 0 && started(path) < opened ? total - before(path, opened) : total; + }); const total = spent.reduce((sum, tokens) => sum + tokens, 0); const squares = spent.reduce((sum, tokens) => sum + tokens * tokens, 0); windows.push({ used, resets: resets * 1000, expected: total > 0 ? (used * squares) / (total * total) : 0 }); } - return windows; + return { id: `codex ${codexIdentity(home)}`, windows }; +} + +/** + * What tells one Codex account from another: the account ID its login stores in `home`, which two + * homes logged into one account share, or the home itself for a login without one, such as an API key. + */ +function codexIdentity(home: string): string { + let auth: { tokens?: { account_id?: unknown } | null } | null; + try { + auth = JSON.parse(readFileSync(join(home, "auth.json"), "utf8")) as typeof auth; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return realpathSync(home); + } + const id = auth?.tokens?.account_id; + return typeof id === "string" && id !== "" ? id : realpathSync(home); } /** The agents' ledger of this host, beside the gate's. */ @@ -218,12 +312,15 @@ const agentsDirectory = (gate: string) => join(dirname(dirname(gate)), "agents", */ export function processAdmission(notice: (text: string) => void, env = process.env): AdmissionDependencies { const gate = claimDirectory(); + const rollouts: RolloutCache = new Map(); return { limits: hostLimits(), gate: fileHost(gate), agents: fileHost(agentsDirectory(gate)), account: (harness) => - harness === "codex" ? codexAccount(env.CODEX_HOME || join(homedir(), ".codex"), Date.now()) : null, + harness === "codex" + ? codexAccount(env.CODEX_HOME || join(homedir(), ".codex"), Date.now(), rollouts) + : null, notice, }; } diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 172261ee..26670aa1 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -27,7 +27,7 @@ export interface Admission { queued?: boolean; start?: string; cores?: number; - /** In the agents' ledger (see admission.ts), the harness of the agent this entry admitted. */ + /** In the agents' ledger (see admission.ts), the account of the agent this entry admitted, or its harness. */ agent?: string; } diff --git a/src/cli/lines.ts b/src/cli/lines.ts new file mode 100644 index 00000000..d54264e5 --- /dev/null +++ b/src/cli/lines.ts @@ -0,0 +1,27 @@ +// Reading a log too large to hold, line by line. +import { closeSync, openSync, readSync } from "node:fs"; + +/** + * Visit each line of the file at `path`, reading it in bounded chunks since logs reach gigabytes. The last line is + * marked incomplete when no newline ends it, as while its session is still writing it. Reading stops once `visit` + * returns false. + */ +export function eachLine(path: string, visit: (line: string, complete: boolean) => boolean | void): void { + const fd = openSync(path, "r"); + try { + const chunk = Buffer.alloc(1 << 22); + let rest = Buffer.alloc(0); + for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) { + const data = rest.length ? Buffer.concat([rest, chunk.subarray(0, read)]) : chunk.subarray(0, read); + let start = 0; + for (let end = data.indexOf(10); end >= 0; end = data.indexOf(10, start)) { + if (end > start && visit(data.toString("utf8", start, end), true) === false) return; + start = end + 1; + } + rest = Buffer.from(data.subarray(start)); + } + if (rest.length) visit(rest.toString("utf8"), false); + } finally { + closeSync(fd); + } +} diff --git a/src/cli/main.ts b/src/cli/main.ts index a8639f1a..8bf4a2dd 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1094,7 +1094,7 @@ export function main(argv: string[], out: Output): number | Promise { "adds no gated load until its first build; an agent start or resume runs counts only until it exits. Agents " + "already running are never paused.\n\n" + "The account has room while, in each usage window its harness reports, the usage spent plus what this " + - "agent and the harness's agents in flight on this host are expected to spend stays within the window. " + + "agent and the account's agents in flight on this host are expected to spend stays within the window. " + "Codex reports its windows in its session logs, read from $CODEX_HOME or ~/.codex, and an agent's " + "expected share is the token-weighted mean share of this host's sessions since the window opened. " + "Claude Code reports its windows to no command, so it is admitted on the host alone, saying so.\n\n" + diff --git a/src/cli/messages.ts b/src/cli/messages.ts index a870f2a0..41076ba1 100644 --- a/src/cli/messages.ts +++ b/src/cli/messages.ts @@ -8,7 +8,8 @@ import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { discoveryEnv } from "./git"; import { githubRepository } from "./github"; -import { claudeLogs, eachLine, parseLine } from "./usage"; +import { eachLine } from "./lines"; +import { claudeLogs, parseLine } from "./usage"; /** One message a person typed. */ export interface Message { diff --git a/src/cli/usage.ts b/src/cli/usage.ts index d2e24089..7d1bdfac 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -2,7 +2,7 @@ // the work went by the PR's own record. ccusage reads and prices the sessions; quest ties each to a PR by the // branch its log names, reading only that metadata and never a prompt or reply. import { spawnSync } from "node:child_process"; -import { closeSync, existsSync, openSync, readdirSync, readSync } from "node:fs"; +import { existsSync, readdirSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { type Size, SIZES, handedBack, questBranch } from "../core"; @@ -10,6 +10,7 @@ import { recorded } from "../github/models"; import { type ClaimDependencies, repositoryPath, upstream } from "./claims"; import { discoveryEnv, git } from "./git"; import { fixes } from "./land-fixer"; +import { eachLine } from "./lines"; import { MERGER } from "./land-watch-fix"; import { type GitHub, type ObjectValue, type Repository, githubRepository, link, requireRemote } from "./github"; import { type OnlineReview, readOnlineReviews } from "./online-review"; @@ -210,31 +211,6 @@ export function parseCcusage(text: string): Priced[] { }); } -/** - * Visit each line of the file at `path`, reading it in bounded chunks since logs reach gigabytes. The last line is - * marked incomplete when no newline ends it, as while its session is still writing it. Reading stops once `visit` - * returns false. - */ -export function eachLine(path: string, visit: (line: string, complete: boolean) => boolean | void): void { - const fd = openSync(path, "r"); - try { - const chunk = Buffer.alloc(1 << 22); - let rest = Buffer.alloc(0); - for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) { - const data = rest.length ? Buffer.concat([rest, chunk.subarray(0, read)]) : chunk.subarray(0, read); - let start = 0; - for (let end = data.indexOf(10); end >= 0; end = data.indexOf(10, start)) { - if (end > start && visit(data.toString("utf8", start, end), true) === false) return; - start = end + 1; - } - rest = Buffer.from(data.subarray(start)); - } - if (rest.length) visit(rest.toString("utf8"), false); - } finally { - closeSync(fd); - } -} - /** One JSON record of a log line, or null for an incomplete last line. */ export function parseLine(path: string, line: string, complete: boolean): Record | null { try { diff --git a/test/admission.test.ts b/test/admission.test.ts index d1ad769a..dee99bb7 100644 --- a/test/admission.test.ts +++ b/test/admission.test.ts @@ -1,5 +1,5 @@ -import { mkdirSync, utimesSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; +import { mkdirSync, realpathSync, utimesSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { expect, test } from "vitest"; import { type AdmissionDependencies, PENDING_MS, type Window, admit, codexAccount } from "../src/cli/admission"; import { type Admission, type Host, RECENT_MS, SETTLE_MS, fileHost, processStart } from "../src/cli/gate"; @@ -11,7 +11,14 @@ const me = { pid: process.pid, start: processStart(process.pid) }; /** A host of 8 CPUs with real ledgers, and load, memory, a clock, and process liveness the test controls. */ function host(windows: Window[] | null = null) { - const state = { now: 10 * PENDING_MS, load: 0, memory: Infinity, dead: new Set(), stop: Infinity }; + const state = { + now: 10 * PENDING_MS, + load: 0, + memory: Infinity, + dead: new Set(), + stop: Infinity, + account: "main", + }; const sleeps: number[] = []; const notices: string[] = []; const ledger = (directory: string): Host => ({ @@ -30,7 +37,7 @@ function host(windows: Window[] | null = null) { limits: { load: 8, memory: 4 * GiB }, gate: ledger(tempDir()), agents: ledger(tempDir()), - account: () => windows, + account: (harness) => windows && { id: `${harness} ${state.account}`, windows }, notice: (text) => notices.push(text), }; const gated = (admissions: Admission[]) => deps.gate.publish(deps.gate.ledger(), admissions); @@ -120,8 +127,11 @@ test("the account admits an agent only while its windows cover it and the agents // The second window reset before now, so it holds nothing; 70 + 10 per agent covers three. expect([await atOnce("codex", 1), await atOnce("codex", 2), await atOnce("codex", 3)]).toEqual([true, true, true]); expect(await admit("codex", deps, signal)).toEqual({ resets }); - // Another harness's agents spend another account. + // Another account's agents spend their own windows, on the same harness or another. + state.account = "other"; + expect(await atOnce("codex", 4)).toBe(true); expect(await atOnce("claude")).toBe(true); + state.account = "main"; // An agent that ended is no longer in flight, however recently it started. state.dead.add(2); expect(await atOnce("codex")).toBe(true); @@ -132,57 +142,119 @@ test("the account admits an agent only while its windows cover it and the agents expect(notices).toEqual([]); }); +test("usage spent while an agent waits for the host refuses it, and an account that changes under it fails", async () => { + const resets = 20 * PENDING_MS; + const windows = [{ used: 50, resets, expected: 10 }]; + const { state, deps, sleeps } = host(windows); + state.memory = GiB; + // Other agents spend the account during the second wait, and memory never frees. + deps.agents.sleep = async (ms) => { + sleeps.push(ms); + state.now += ms; + if (sleeps.length === 2) windows[0].used = 95; + }; + expect(await admit("codex", deps, signal)).toEqual({ resets }); + expect(sleeps).toHaveLength(2); + windows[0].used = 50; + deps.agents.sleep = async () => { + state.account = "other"; + }; + await expect(admit("codex", deps, signal)).rejects.toThrow("stopped reporting the usage of the account"); +}); + test("a harness that reports no usage is admitted on the host alone, saying so", async () => { const { atOnce, notices } = host(null); expect(await atOnce("claude")).toBe(true); expect(notices).toEqual(["claude reports no account usage here, so only the host is checked"]); }); -/** A Codex rollout in `home` last written at `modified` ms, ending with a `token_count` event. */ -function rollout(home: string, name: string, modified: number, tokens: number, limits: unknown, tail = "") { - const dir = join(home, "sessions", "2026", "11", "12"); - mkdirSync(dir, { recursive: true }); - const path = join(dir, `rollout-${name}.jsonl`); - const event = { - type: "event_msg", - payload: { type: "token_count", info: { total_token_usage: { total_tokens: tokens } }, rate_limits: limits }, - }; - writeFileSync(path, `{"type":"session_meta"}\n${JSON.stringify(event)}\n${tail}`); - utimesSync(path, modified / 1000, modified / 1000); +const minute = 60_000; +const now = 1_000_000 * minute; + +/** + * A Codex rollout at `path` under `home`, last written at `modified` ms, holding a `token_count` + * event per `[time, total tokens, rate limits]`, then `tail`. + */ +function rollout(home: string, path: string, modified: number, events: [number, number, unknown][], tail = "") { + const file = join(home, path.includes("/") ? path : `sessions/2026/11/12/rollout-${path}.jsonl`); + mkdirSync(dirname(file), { recursive: true }); + const lines = events.map(([at, tokens, limits]) => + JSON.stringify({ + timestamp: new Date(at).toISOString(), + type: "event_msg", + payload: { + type: "token_count", + info: { total_token_usage: { total_tokens: tokens } }, + rate_limits: limits, + }, + }), + ); + writeFileSync(file, `{"type":"session_meta"}\n${lines.join("\n")}\n${tail}`); + utimesSync(file, modified / 1000, modified / 1000); } +const week = { used_percent: 40, window_minutes: 10_080, resets_at: (now + 1000 * minute) / 1000 }; +const hours = { used_percent: 10, window_minutes: 300, resets_at: (now + 100 * minute) / 1000 }; +const old = { primary: { ...week, used_percent: 5 }, secondary: null }; + test("Codex's windows come from its newest report, with an agent's share from this host's sessions", () => { const home = tempDir(); - const minute = 60_000; - const now = 1_000_000 * minute; - const week = { used_percent: 40, window_minutes: 10_080, resets_at: (now + 1000 * minute) / 1000 }; - const hours = { used_percent: 10, window_minutes: 300, resets_at: (now + 100 * minute) / 1000 }; - const old = { primary: { ...week, used_percent: 5 }, secondary: null }; // Written before the 5-hour window opened, inside the week's. - rollout(home, "a", now - 400 * minute, 300, old); - rollout(home, "b", now - 50 * minute, 100, old); - // The newest rollout's last line is still being written, so its last whole report counts. + rollout(home, "a", now - 400 * minute, [[now - 400 * minute, 300, old]]); + // An archived session still spent the window, and one seen in both trees counts once. + rollout(home, "archived_sessions/rollout-b.jsonl", now - 50 * minute, [[now - 50 * minute, 100, old]]); + rollout(home, "sessions/2026/11/13/rollout-b.jsonl", now - 50 * minute, [[now - 50 * minute, 100, old]]); + // The newest report's rollout has a last line still being written, so its last whole one counts. rollout( home, "c", - now - minute, - 100, - { primary: week, secondary: hours }, + now - 2 * minute, + [[now - 2 * minute, 100, { primary: week, secondary: hours }]], '{"type":"event_msg","payload":{"type":"token_c', ); - const [weekly, hourly] = codexAccount(home, now)!; + // A rollout written since, by a tool's output, holds only an older report. + rollout(home, "d", now - minute, [[now - 30 * minute, 0, old]]); + const { id, windows } = codexAccount(home, now)!; + expect(id).toBe(`codex ${realpathSync(home)}`); // Sessions of 300, 100, and 100 tokens: a token's session holds (9 + 1 + 1) / 25 of the window's spend. - expect(weekly).toEqual({ used: 40, resets: now + 1000 * minute, expected: (40 * 11) / 25 }); - expect(hourly).toEqual({ used: 10, resets: now + 100 * minute, expected: 10 / 2 }); + expect(windows).toEqual([ + { used: 40, resets: now + 1000 * minute, expected: (40 * 11) / 25 }, + { used: 10, resets: now + 100 * minute, expected: 10 / 2 }, + ]); // A window that reset since the report is unknown, not spent. - expect(codexAccount(home, now + 200 * minute)).toEqual([expect.objectContaining({ used: 40 })]); + expect(codexAccount(home, now + 200 * minute)!.windows).toEqual([expect.objectContaining({ used: 40 })]); +}); + +test("a session resumed across a window's reset counts only its tokens since", () => { + const home = tempDir(); + const opened = now + 100 * minute - 300 * minute; + // 900 of its 1000 tokens were spent before the 5-hour window opened. + rollout(home, "1970-06-01T00-00-00-a", now - minute, [ + [opened - 500 * minute, 400, old], + [opened - minute, 900, old], + [now - minute, 1000, { primary: hours, secondary: null }], + ]); + rollout(home, "b", now - 10 * minute, [[now - 10 * minute, 100, old]]); + // Two sessions of 100 tokens each in the window: half of its 10 points apiece. + expect(codexAccount(home, now)!.windows).toEqual([{ used: 10, resets: now + 100 * minute, expected: 5 }]); +}); + +test("two Codex homes are one account only when their logins name the same one", () => { + const [first, second] = [tempDir(), tempDir()]; + for (const home of [first, second]) rollout(home, "a", now, [[now, 1, { primary: week, secondary: null }]]); + expect(codexAccount(first, now)!.id).not.toBe(codexAccount(second, now)!.id); + for (const home of [first, second]) writeFileSync(join(home, "auth.json"), '{"tokens":{"account_id":"acct"}}'); + expect([codexAccount(first, now)!.id, codexAccount(second, now)!.id]).toEqual(["codex acct", "codex acct"]); + // An API key's login names no account. + writeFileSync(join(second, "auth.json"), '{"OPENAI_API_KEY":"sk","tokens":null}'); + expect(codexAccount(second, now)!.id).toBe(`codex ${realpathSync(second)}`); }); test("Codex without a usage report gives no signal, and an unreadable one fails", () => { const home = tempDir(); expect(codexAccount(home, 0)).toBeNull(); - rollout(home, "a", 1000, 100, null); + rollout(home, "a", 1000, [[1000, 100, null]]); expect(codexAccount(home, 0)).toBeNull(); - rollout(home, "b", 2000, 100, { primary: { used_percent: "40%" }, secondary: null }); + rollout(home, "b", 2000, [[2000, 100, { primary: { used_percent: "40%" }, secondary: null }]]); expect(() => codexAccount(home, 0)).toThrow("Codex reported a primary usage window quest cannot read"); }); From 7d4c6ee88a455bab986dfe804dd15500e9a7bfd0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 08:50:52 -0800 Subject: [PATCH 5/6] refactor(agent): admit on the host alone; drop Codex account usage Maintainer decision (2026-11-13): keep the host half, which reuses the gate's room and pending-count logic, and drop the Codex account-usage half. It parsed Codex session logs and could not see the Claude quota behind the "credentials cooling down" errors. `quest agent admit` loses --harness and exit 3, start and resume no longer print `outcome limited` on admission, and the agents' ledger no longer tags entries by account. Deletes the follow-up quests agent-usage.md and agent-admission-measure.md. Also deletes quest/m0/gate-memory.md, whose PR #693 the maintainer closed: the host's systemd-oomd swap kill owns memory, and the gate keeps at most OOMPolicy=kill plus reporting an oom-kill result. Co-Authored-By: Claude Opus 5.5 --- README.md | 33 ++-- assets/AGENTS.md | 3 +- quest/m0/agent-admission-measure.md | 33 ---- quest/m0/agent-usage.md | 48 ----- quest/m0/gate-memory.md | 58 ------ src/cli/admission.ts | 275 ++-------------------------- src/cli/agent.ts | 40 ++-- src/cli/gate.ts | 5 +- src/cli/lines.ts | 27 --- src/cli/main.ts | 30 +-- src/cli/messages.ts | 3 +- src/cli/usage.ts | 28 ++- test/admission.test.ts | 164 +---------------- test/agent.test.ts | 31 +--- 14 files changed, 97 insertions(+), 681 deletions(-) delete mode 100644 quest/m0/agent-admission-measure.md delete mode 100644 quest/m0/agent-usage.md delete mode 100644 quest/m0/gate-memory.md delete mode 100644 src/cli/lines.ts diff --git a/README.md b/README.md index 5008dc82..230c3f4f 100644 --- a/README.md +++ b/README.md @@ -1663,7 +1663,7 @@ quest agent start --harness claude --model claude-opus-5-5 --effort high --permi quest agent resume --harness claude --model claude-opus-5-5 --effort high --permission-mode auto .worktrees/foo "Take the recommendation." quest agent stop .worktrees/foo quest agent stop --all -quest agent admit --harness claude +quest agent admit ``` Codex starts and resumes only in a checkout registered by `quest worktree create`, @@ -1677,30 +1677,19 @@ pruned dependencies when the next turn needs them. Claude continues to run directly in the supplied checkout. Codex desktop threads use the desktop app's native archival instead. -Start and resume first wait until the host and the account can carry one more -agent, before claiming the worktree, and a coordinator runs `quest agent admit ---harness ` before it starts or resumes a sub-agent of its own: - -- The host has room while `quest gate` would admit a command at once, counting - every command queued for it and, as one recent admission each, every agent - admitted in the last 5 minutes: an agent adds no gated load until its first - build, and three quarters of those that build at all reach it by then. An - agent `start` or `resume` runs stops counting when it exits. As in the gate, - a host loaded only by work no gate holds still starts one agent at a time. -- The account has room while, in each usage window its harness reports, the - usage spent plus what this agent and the account's agents in flight on this - host are expected to spend stays within the window. Codex reports its windows - in its session logs, read afresh at each decision; an agent's expected share is the token-weighted mean - share of this host's sessions since the window opened, so a host with no - history there expects none. Claude Code reports its windows only to an - interactive session's status line, so its agents are admitted on the host - alone, and each admission says so. +Start and resume first wait until the host can carry one more agent, before +claiming the worktree, and a coordinator runs `quest agent admit` before it +starts or resumes a sub-agent of its own. The host has room while `quest gate` +would admit a command at once, counting every command queued for it and, as one +recent admission each, every agent admitted in the last 5 minutes: an agent adds +no gated load until its first build, and three quarters of those that build at +all reach it by then. An agent `start` or `resume` runs stops counting when it +exits. As in the gate, a host loaded only by work no gate holds still starts one +agent at a time. Admission only delays the next agent: none already running is paused or stopped, and `quest agent stop` does not see a launch still waiting, which ends -when its command is cancelled. `admit` exits 0 once admitted, and all three exit -3 at once, naming the reset, when the account cannot cover the agent; start and -resume then print `outcome limited` and no session. +when its command is cancelled. Options go before the worktree, and the prompt or message follows it, so text starting with `-` is never read as an option. The agent runs in the worktree diff --git a/assets/AGENTS.md b/assets/AGENTS.md index 2cd6566c..199341d6 100644 --- a/assets/AGENTS.md +++ b/assets/AGENTS.md @@ -192,8 +192,7 @@ Relay no interim report: resume a sub-agent that ended its turn on one, telling A long-running coordinator lets compaction bound its context: what must outlast it lives in the forge and the quest tree, and a PR's state is read afresh before acting, not trusted from earlier turns. Start each sub-agent in the background with its own isolated worktree, since agents cannot write to a worktree their parent created. -Before starting or resuming one, run `quest agent admit --harness ` in the background and go on once it exits 0, handling reports meanwhile: it waits until the host has room for another agent. `quest agent start` and `resume` admit themselves. -Its exit 3 means the account's reported usage cannot cover another agent until the reset it names: a usage limit, handled as below. Report once a harness it says reports no usage. +Before starting or resuming one, run `quest agent admit` in the background and go on once it exits 0, handling reports meanwhile: it waits until the host has room for another agent. `quest agent start` and `resume` admit themselves. A sub-agent reports only when done or blocked, and never waits on GitHub CI, reviews, or merges; stay silent on its interim notifications. As each reports, explain its result in a few lines: its PR, whether it is still a draft, and its open decisions with recommendations. Once two agents report the same blocker, such as a check failure that reproduces on the base or a reviewer outage, name it with its evidence in each later agent's brief, and report it once. diff --git a/quest/m0/agent-admission-measure.md b/quest/m0/agent-admission-measure.md deleted file mode 100644 index a035d335..00000000 --- a/quest/m0/agent-admission-measure.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] Agent admission is measured on a real fan-out - -## Goal - -A `/quest-spawn` fan-out on the maintainer's host, run with agent admission, -is compared with the three weeks before it: usage-limit errors, agents lost -to them, gate waits over 10 and 30 minutes, and total wall time. The numbers -go in this quest's PR, with any change to admission they call for. - -## Plan - -Baseline, from transcripts on the maintainer's three Linux hosts, 2026-10-22 -to 11-12: fan-outs of 20 to 26 sub-agents; 157 "All credentials ... cooling -down" errors on one host; 156 background agents "terminated early due to an -API error"; and of 2,482 gated commands on one host, 191 waited over 10 -minutes, 77 over 30, and 11 were still queued at the 55-minute timeout. - -[moq-dev/quest#692](https://github.com/moq-dev/quest/pull/692) could not run -one: every host's load was above its CPU count, with gated commands queued, -so admission held every agent, which is its job and no fan-out. - -Agent's choices, for the implementer to revisit: - -- Give each agent its own `QUEST_GATE_RECORD` file, and read the waits from - those rather than from transcripts. -- Check that the 5 minutes an admitted agent counts against the host - (`PENDING_MS` in `src/cli/admission.ts`) still covers three quarters of the - agents that gate anything, and that waves of admissions do not arrive at - the gate together. - -## Required - -- [The gate records each run for the process that started the agent](/quest/m0/gate-record.md) - each agent's gate waits, read without matching transcripts diff --git a/quest/m0/agent-usage.md b/quest/m0/agent-usage.md deleted file mode 100644 index 0f674d87..00000000 --- a/quest/m0/agent-usage.md +++ /dev/null @@ -1,48 +0,0 @@ -# [M] Agent admission reads Claude's usage and sizes an agent's share - -## Goal - -`quest agent admit`, `start`, and `resume` refuse a Claude agent its account's -usage cannot cover, as they already do for Codex, and expect more of the -account from a large quest's agent than from a small one's. - -## Plan - -Found 2026-11-12 while building agent admission -([moq-dev/quest#692](https://github.com/moq-dev/quest/pull/692)), which -checks the host for every harness and the account for Codex alone: - -- Claude Code 2.1.322 gives no command its usage windows. Its docs name them - only in the JSON a `statusLine` command receives (`rate_limits.five_hour` - and `rate_limits.seven_day`, each with `used_percentage` and `resets_at`), - for claude.ai subscribers. A headless `claude -p --output-format - stream-json --verbose` turn on the CLI's own login printed no - `rate_limit_event`, which the SDK docs say comes only when a limit's status - changes, and no transcript or file under `~/.claude` holds the windows. So - admission says Claude reports no usage and checks the host alone. -- Sessions through a credential-pool gateway (`ANTHROPIC_BASE_URL`) have no - single account: a pool's windows are its credentials', which only the - gateway knows, and asking it is a non-goal of admission. -- An agent's expected share is the token-weighted mean share of the host's - sessions since the window opened, whatever its quest's size. `quest - usage`'s per-size history, which the plan named, took 21 seconds and a - listing of the repository's PRs per call on the maintainer's host, and - attributed 29 of 964 sessions to a PR, with no row for L or XL. - -Agent's choices, for the implementer to revisit: - -- Read Claude's windows from a status line: `quest setup` installs a - `statusLine` command that records the `rate_limits` it is given, chaining - to the one the user already has, and admission reads the newest record. - It covers only hosts where an interactive session runs on the account. -- For sizes, take `--size` on `admit`, `start`, and `resume`, and scale the - host's mean share by the size's share of spend where `quest usage` has - history for it, read from a cache a coordinator refreshes once per run. - -## Waiting - -- Maintainer: whether an agent's expected share stays the host's mean for every size (recommended: the per-size history is too thin to change a decision, and a wrong size estimate refuses agents the account could carry) or takes the quest's size as above. An agent cannot pick: the plan decided per-size history before the data showed how little of it there is. Check: this item is answered in the Plan. - -## Related - -- [Cut agent token waste](/quest/m0/token-waste/README.md) - the other half of what an account's usage buys diff --git a/quest/m0/gate-memory.md b/quest/m0/gate-memory.md deleted file mode 100644 index 41da5227..00000000 --- a/quest/m0/gate-memory.md +++ /dev/null @@ -1,58 +0,0 @@ -# [S] A gated command cannot exhaust the host's memory - -## Goal - -On Linux hosts where the gate owns each command in a systemd scope, a -gated command that grows past its memory ceiling is throttled and then -killed as a whole group, instead of the kernel's OOM killer picking a -victim elsewhere on the host. Other gated commands, agent sessions, and -host services keep running. - -Non-goals: host-wide swap or oomd settings, which kixel-ai/nixos's -quest tree plans for the maintainer's host. - -## Plan - -Found 2026-11-12 in read-only audits of the maintainer's three Linux -hosts: the kernel OOM killer fired 5 times in 10 days on one host, killing a -47 GB `rust2-ts` inside a gate scope, a 36 GB `lean`, and a 38 GB gated -`java` while a different process was the one allocating. earlyoom killed -34 processes in a week on another, including HOL Light, lean, and an -Apalache run with a 16 GB heap. Gate scopes start with no memory limit -(`systemd-run` in `src/cli/supervise.ts`). - -Decided 2026-11-12 (maintainer, /quest-plan): - -- ✅ The gate's scopes get a memory ceiling and kill the whole group on - OOM (rejected: leaving it to host-wide oomd alone, which picks victims by - pressure, not by which command overran; rules telling agents to cap - heaps themselves). -- Milestone: ✅ m0 (rejected: m1). - -What the decision requires, found in review: - -- The ceilings of commands running at once must fit the host together: - per-command ceilings each sized from the memory available at admission - overcommit, since several commands admitted together each see the same - free memory. Partition one host budget (memory less a reserve for - sessions and services) across admitted commands, as the gate's memory - admission already counts each command's share (`COMMAND_MEMORY` in - `src/cli/gate.ts`), and admit a command only when its ceiling fits. -- A nested gate's scope is a sibling bound to its enclosing scope - (`BindsTo`), not a child cgroup, so it does not inherit the parent's - limit: its usage must come out of the enclosing command's share. - -Agent's choices, for the implementer to revisit: - -- Set `MemoryHigh` below `MemoryMax`, so a command slows under reclaim - before it is killed, and `OOMPolicy=kill`, so an OOM kill inside the - scope stops the whole command rather than leaving its wrappers running. -- Size a command's share from its measured peak once the gate has seen - it, rather than a constant (moq-dev/quest#673 dropped the matching CPU - plan); a command - that needs more fails loud with the scope's OOM event in the gate's - error, so the agent learns why. - -## Hosts - -- linux - systemd scopes carry the memory limits diff --git a/src/cli/admission.ts b/src/cli/admission.ts index 475edc53..c1871db4 100644 --- a/src/cli/admission.ts +++ b/src/cli/admission.ts @@ -1,9 +1,6 @@ // Agent admission: a coordinator starts or resumes an agent only while the host's gate would admit -// work at once and the account's reported usage covers the agents in flight. It only delays the next -// start: agents already running are never paused or stopped. +// work at once. It only delays the next start: agents already running are never paused or stopped. import { randomUUID } from "node:crypto"; -import { closeSync, fstatSync, openSync, readFileSync, readSync, readdirSync, realpathSync, statSync } from "node:fs"; -import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; import { @@ -18,7 +15,6 @@ import { processStart, room, } from "./gate"; -import { eachLine } from "./lines"; /** * How long an admitted agent counts against the host's headroom, as one recent admission of the @@ -28,23 +24,6 @@ import { eachLine } from "./lines"; */ export const PENDING_MS = 5 * 60_000; -/** One usage window an account's harness reports. */ -export interface Window { - /** The percentage of the window spent. */ - used: number; - /** When the window resets, in epoch milliseconds. */ - resets: number; - /** The percentage one more agent is expected to spend, from this host's sessions in the window. */ - expected: number; -} - -/** An account's usage as its harness last reported it. */ -export interface Account { - /** Names the account among every harness's, so its agents in flight count against it alone. */ - id: string; - windows: Window[]; -} - /** What `admit` reads and writes, replaceable in tests. */ export interface AdmissionDependencies { limits: Limits; @@ -52,17 +31,10 @@ export interface AdmissionDependencies { gate: Host; /** The agents' ledger; only its clock, ledger, liveness, and sleep are used. */ agents: Host; - /** The account `harness` runs on with its usage as of now, or null when the harness reports none. */ - account(harness: string): Account | null; /** A line for the user. */ notice(text: string): void; } -/** The account cannot cover one more agent until `resets`, in epoch milliseconds. */ -export interface Refusal { - resets: number; -} - const waits: Record = { load: "the load average to fall", memory: "free memory", @@ -70,51 +42,29 @@ const waits: Record = { }; /** - * Wait until the host can carry one more agent of `harness`, then record it. The host has room while - * the gate would admit a command at once (see `room`), counting each agent admitted in the last - * PENDING_MS whose launcher, if it has one, still runs. The account has room while, in every window - * its harness reports, the usage spent plus what this agent and the account's agents in flight are - * expected to spend stays within the window; in flight are the agents whose launcher still runs, and - * those without one admitted in the last PENDING_MS. The usage is read afresh at every decision, - * since agents spend it while this one waits for the host. Returns null once admitted, or why the - * account cannot cover it, which no wait here clears. A harness that reports no usage is admitted - * on the host alone, saying so. + * Wait until the host can carry one more agent, then record it. The host has room while the gate + * would admit a command at once (see `room`), counting each agent admitted in the last PENDING_MS + * whose launcher, if it has one, still runs. * * `held` is the process that runs the agent, as `quest agent start` does; without it, as for a * coordinator's own sub-agent, nothing reports the agent's end. */ -export async function admit( - harness: string, - deps: AdmissionDependencies, - signal: AbortSignal, - held?: number, -): Promise { - // Decided once, at the start: a setup either reports its usage or does not. - const first = deps.account(harness); - if (first === null) deps.notice(`${harness} reports no account usage here, so only the host is checked`); +export async function admit(deps: AdmissionDependencies, signal: AbortSignal, held?: number): Promise { const holder = held === undefined ? {} : { pid: held, start: processStart(held) }; const shown = new Set(); - for (let account = first; ; account = first && deps.account(harness)) { + for (;;) { signal.throwIfAborted(); - if (first !== null && account?.id !== first.id) - throw new Error(`${harness} stopped reporting the usage of the account this admission began on`); const ledger = deps.agents.ledger(); const now = deps.agents.now(); - const kept = ledger.admissions - .filter((entry) => (entry.pid === undefined ? entry.at > now - PENDING_MS : deps.agents.running(entry))) - // A clock stepped backward dates an entry in the future; count it from now instead. - .map((entry) => (entry.at > now ? { ...entry, at: now } : entry)); - const id = account?.id ?? harness; - const flying = kept.filter((entry) => entry.agent === id).length; - const short = (account?.windows ?? []) - .filter((window) => window.resets > now && window.used + (flying + 1) * window.expected > 100) - .map((window) => window.resets); - if (short.length > 0) return { resets: Math.max(...short) }; - const pending = kept.filter((entry) => entry.at > now - PENDING_MS).length; - const reason = room(deps.limits, deps.gate, pending); + const pending = ledger.admissions.filter( + (entry) => entry.at > now - PENDING_MS && (entry.pid === undefined || deps.agents.running(entry)), + ); + const reason = room(deps.limits, deps.gate, pending.length); if (reason === null) { - const entry: Admission = { id: randomUUID(), at: now, ...holder, agent: id }; - if (deps.agents.publish(ledger, [...kept, entry])) return null; + // A clock stepped backward dates an entry in the future; count it from now instead. + const kept = pending.map((entry) => (entry.at > now ? { ...entry, at: now } : entry)); + const entry: Admission = { id: randomUUID(), at: now, ...holder }; + if (deps.agents.publish(ledger, [...kept, entry])) return; continue; } if (!shown.has(reason)) { @@ -125,204 +75,13 @@ export async function admit( } } -/** How much of a rollout's end is read for its last usage report: a report is about a kilobyte. */ -const TAIL = 256 * 1024; - -/** One `token_count` event of a Codex rollout: when, the session's tokens so far, and the account's windows. */ -interface Count { - at: number; - tokens: number; - limits: Record | null; -} - -/** The `token_count` event on a rollout's `line`, if it is one and whole. */ -function count(line: string): Count | undefined { - if (!line.includes('"token_count"')) return undefined; - let event: { timestamp?: unknown; payload?: Record }; - try { - event = JSON.parse(line) as typeof event; - } catch { - return undefined; // A tail's first line is cut, and a session still writing leaves its last one open. - } - const at = typeof event.timestamp === "string" ? Date.parse(event.timestamp) : NaN; - if (event.payload?.type !== "token_count" || Number.isNaN(at)) return undefined; - const total = (event.payload.info as { total_token_usage?: { total_tokens?: unknown } } | null)?.total_token_usage; - const limits = event.payload.rate_limits; - return { - at, - tokens: typeof total?.total_tokens === "number" ? total.total_tokens : 0, - limits: typeof limits === "object" && limits !== null ? (limits as Record) : null, - }; -} - -/** The last `token_count` event in the last TAIL bytes of the Codex rollout at `path`, if any. */ -function lastCount(path: string): Count | undefined { - const fd = openSync(path, "r"); - let text: string; - try { - const size = fstatSync(fd).size; - const buffer = Buffer.alloc(Math.min(size, TAIL)); - readSync(fd, buffer, 0, buffer.length, size - buffer.length); - text = buffer.toString("utf8"); - } finally { - closeSync(fd); - } - const lines = text.split("\n"); - for (let index = lines.length - 1; index >= 0; index--) { - const found = count(lines[index]); - if (found) return found; - } - return undefined; -} - -/** The tokens the session in the rollout at `path` had spent by `time`, from its last report before then. */ -function tokensBefore(path: string, time: number): number { - let tokens = 0; - eachLine(path, (line) => { - const found = count(line); - if (found === undefined) return; - if (found.at >= time) return false; - tokens = found.tokens; - }); - return tokens; -} - -/** - * What earlier reads of each rollout found, by path, so an admission that waits reads only what - * changed: its last report as of `modified`, and its tokens before each window's opening. - */ -export type RolloutCache = Map }>; - -/** When a rollout's session started, from the local time its file name carries, or 0 when it has none. */ -function started(path: string): number { - const [, year, month, day, hour, minute, second] = - /rollout-(\d{4})-(\d\d)-(\d\d)T(\d\d)-(\d\d)-(\d\d)-/.exec(basename(path)) ?? []; - return year === undefined ? 0 : new Date(+year, +month - 1, +day, +hour, +minute, +second).getTime(); -} - -/** - * The usage windows Codex last reported for the account logged in at `home`, from the newest - * `rate_limits` event among its session rollouts, archived ones included, or null when none holds - * one. A window's capacity in tokens is not reported, so an agent's expected share comes from the - * account's own history: the sessions this host wrote since the window opened spent `used` between - * them, counting only their tokens since then, and an agent is expected to spend the share of the - * session a token most likely belongs to, their token-weighted mean, since most spend comes from the - * long sessions an agent's is. Other hosts' spend on the account raises `used` without adding - * sessions here, which only makes the estimate more careful. - */ -export function codexAccount(home: string, now: number, cache: RolloutCache = new Map()): Account | null { - const found = new Map(); - const walk = (dir: string) => { - let entries; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return; - throw error; - } - for (const entry of entries) { - const path = join(dir, entry.name); - if (entry.isDirectory()) walk(path); - // Archiving moves a rollout, so a scan racing it may see both copies of one session. - else if (entry.name.endsWith(".jsonl") && !found.has(entry.name)) - found.set(entry.name, { path, modified: statSync(path).mtimeMs }); - } - }; - walk(join(home, "sessions")); - walk(join(home, "archived_sessions")); - const rollouts = [...found.values()].sort((a, b) => b.modified - a.modified); - const modifiedAt = new Map(rollouts.map(({ path, modified }) => [path, modified])); - const read = (path: string) => { - const modified = modifiedAt.get(path)!; - const known = cache.get(path); - if (known?.modified === modified) return known; - const entry = { modified, last: lastCount(path), before: known?.before ?? new Map() }; - cache.set(path, entry); - return entry; - }; - const lastOf = (path: string) => read(path).last; - // What a session had spent by a past time never changes, however much it writes later. - const before = (path: string, time: number) => { - const known = cache.get(path)?.before.get(time); - if (known !== undefined) return known; - const tokens = tokensBefore(path, time); - read(path).before.set(time, tokens); - return tokens; - }; - // A rollout written after its last report can be newer than one holding a later report, but no - // report is newer than its file, so the search ends at the first file older than the best report. - let reported: Count | undefined; - for (const { path, modified } of rollouts) { - if (reported !== undefined && modified < reported.at) break; - const each = lastOf(path); - if (each?.limits && (reported === undefined || each.at > reported.at)) reported = each; - } - if (!reported?.limits) return null; - const windows: Window[] = []; - for (const name of ["primary", "secondary"]) { - const window = reported.limits[name] as { - used_percent?: unknown; - window_minutes?: unknown; - resets_at?: unknown; - } | null; - if (window == null) continue; - const { used_percent: used, window_minutes: minutes, resets_at: resets } = window; - if (typeof used !== "number" || typeof minutes !== "number" || typeof resets !== "number") - throw new Error(`Codex reported a ${name} usage window quest cannot read: ${JSON.stringify(window)}`); - // A window that has reset since the report holds usage nothing here has read yet. - if (resets * 1000 <= now) continue; - const opened = resets * 1000 - minutes * 60_000; - const spent = rollouts - .filter(({ modified }) => modified >= opened) - .map(({ path }) => { - const total = lastOf(path)?.tokens ?? 0; - // A session resumed across the reset spent part of its tokens in an earlier window. - return total > 0 && started(path) < opened ? total - before(path, opened) : total; - }); - const total = spent.reduce((sum, tokens) => sum + tokens, 0); - const squares = spent.reduce((sum, tokens) => sum + tokens * tokens, 0); - windows.push({ used, resets: resets * 1000, expected: total > 0 ? (used * squares) / (total * total) : 0 }); - } - return { id: `codex ${codexIdentity(home)}`, windows }; -} - -/** - * What tells one Codex account from another: the account ID its login stores in `home`, which two - * homes logged into one account share, or the home itself for a login without one, such as an API key. - */ -function codexIdentity(home: string): string { - let auth: { tokens?: { account_id?: unknown } | null } | null; - try { - auth = JSON.parse(readFileSync(join(home, "auth.json"), "utf8")) as typeof auth; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - return realpathSync(home); - } - const id = auth?.tokens?.account_id; - return typeof id === "string" && id !== "" ? id : realpathSync(home); -} - /** The agents' ledger of this host, beside the gate's. */ const agentsDirectory = (gate: string) => join(dirname(dirname(gate)), "agents", basename(gate)); -/** - * The admission `quest agent` runs with: the host's own limits and ledgers, and each harness's own - * report. Claude Code reports its usage windows only to an interactive session's status line, and a - * headless turn prints none, so it has no signal to read. - */ -export function processAdmission(notice: (text: string) => void, env = process.env): AdmissionDependencies { +/** The admission `quest agent` runs with: the host's own limits and ledgers. */ +export function processAdmission(notice: (text: string) => void): AdmissionDependencies { const gate = claimDirectory(); - const rollouts: RolloutCache = new Map(); - return { - limits: hostLimits(), - gate: fileHost(gate), - agents: fileHost(agentsDirectory(gate)), - account: (harness) => - harness === "codex" - ? codexAccount(env.CODEX_HOME || join(homedir(), ".codex"), Date.now(), rollouts) - : null, - notice, - }; + return { limits: hostLimits(), gate: fileHost(gate), agents: fileHost(agentsDirectory(gate)), notice }; } /** The admission every `quest agent` command runs, held in an object so a test can replace it. */ diff --git a/src/cli/agent.ts b/src/cli/agent.ts index 4b9dab4a..4883e125 100644 --- a/src/cli/agent.ts +++ b/src/cli/agent.ts @@ -24,7 +24,7 @@ import { constants, tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; -import { type Refusal, admission, processAdmission } from "./admission"; +import { admission, processAdmission } from "./admission"; import { discoveryEnv, git } from "./git"; import { cancelled, interrupts } from "./interrupt"; import { LIMIT } from "./limit"; @@ -388,20 +388,16 @@ export interface AgentDependencies { base: string; stdout(text: string): void; stderr(text: string): void; - /** - * Wait until the host can carry one more agent of `harness`, run by process `held` if any, and - * return null, or why its account cannot cover one (see admission.ts). - */ - admit(harness: HarnessName, signal: AbortSignal, held?: number): Promise; + /** Wait until the host can carry one more agent, run by process `held` if any (see admission.ts). */ + admit(signal: AbortSignal, held?: number): Promise; } export const processDependencies = (out: Pick): AgentDependencies => ({ env: process.env, base: agentDirectory(), ...out, - admit: (harness, signal, held) => + admit: (signal, held) => admission.admit( - harness, processAdmission((text) => out.stderr(`quest: ${text}\n`)), signal, held, @@ -409,23 +405,14 @@ export const processDependencies = (out: Pick { +export async function admitAgent(deps: AgentDependencies, held?: number): Promise { const cancel = interrupts(); try { - const refusal = await deps.admit(harness, cancel.signal, held); - if (refusal === null) return undefined; - deps.stderr( - `quest: ${harness}'s reported usage cannot cover another agent; resets ${new Date(refusal.resets).toISOString()}\n`, - ); - return EXIT.limited; + await deps.admit(cancel.signal, held); + return undefined; } catch (error) { if (cancel.signal.aborted) return cancelled(cancel.signal); throw error; @@ -472,7 +459,7 @@ export function boundedLines(limit: number, read: (line: string) => void) { } /** - * Run one agent turn in the foreground, once the host and account can carry it + * Run one agent turn in the foreground, once the host can carry it * (see `admitAgent`): a new session with `prompt`, or * `session` resumed with it. Prints the session id as soon as it is known, * then the outcome and final reply. Returns the exit status. @@ -509,11 +496,8 @@ export async function runAgent( const command = checkouts ? undefined : programCommand(program, args, deps.env); const dir = spawnDirectory(deps.base, worktree); // Waited out before the worktree is claimed, so a queued launch holds nothing. - const refused = await admitAgent(launch.harness, deps, process.pid); - if (refused !== undefined) { - if (refused === EXIT.limited) deps.stdout("outcome limited\n"); - return refused; - } + const refused = await admitAgent(deps, process.pid); + if (refused !== undefined) return refused; const reader = harness.reader(); // Claude Code takes the id this command assigns; Codex announces its own. diff --git a/src/cli/gate.ts b/src/cli/gate.ts index 26670aa1..4e20fa73 100644 --- a/src/cli/gate.ts +++ b/src/cli/gate.ts @@ -27,8 +27,6 @@ export interface Admission { queued?: boolean; start?: string; cores?: number; - /** In the agents' ledger (see admission.ts), the account of the agent this entry admitted, or its harness. */ - agent?: string; } /** One version of the host's admission ledger and queue. Versions only grow; 0 is the empty ledger. */ @@ -259,8 +257,7 @@ function parse(text: string): Admission[] { (entry.pid === undefined || (Number.isSafeInteger(entry.pid) && entry.pid > 0)) && (entry.queued === undefined || entry.queued === true) && (entry.start === undefined || typeof entry.start === "string") && - (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)) && - (entry.agent === undefined || typeof entry.agent === "string"), + (entry.cores === undefined || (Number.isSafeInteger(entry.cores) && entry.cores >= 0)), ); return valid ? (value as Admission[]) : []; } diff --git a/src/cli/lines.ts b/src/cli/lines.ts deleted file mode 100644 index d54264e5..00000000 --- a/src/cli/lines.ts +++ /dev/null @@ -1,27 +0,0 @@ -// Reading a log too large to hold, line by line. -import { closeSync, openSync, readSync } from "node:fs"; - -/** - * Visit each line of the file at `path`, reading it in bounded chunks since logs reach gigabytes. The last line is - * marked incomplete when no newline ends it, as while its session is still writing it. Reading stops once `visit` - * returns false. - */ -export function eachLine(path: string, visit: (line: string, complete: boolean) => boolean | void): void { - const fd = openSync(path, "r"); - try { - const chunk = Buffer.alloc(1 << 22); - let rest = Buffer.alloc(0); - for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) { - const data = rest.length ? Buffer.concat([rest, chunk.subarray(0, read)]) : chunk.subarray(0, read); - let start = 0; - for (let end = data.indexOf(10); end >= 0; end = data.indexOf(10, start)) { - if (end > start && visit(data.toString("utf8", start, end), true) === false) return; - start = end + 1; - } - rest = Buffer.from(data.subarray(start)); - } - if (rest.length) visit(rest.toString("utf8"), false); - } finally { - closeSync(fd); - } -} diff --git a/src/cli/main.ts b/src/cli/main.ts index 8bf4a2dd..bbfc2115 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -26,7 +26,6 @@ import { import { AgentInputError, HARNESSES, - type HarnessName, type Launch, admitAgent, processDependencies, @@ -1033,8 +1032,8 @@ export function main(argv: string[], out: Output): number | Promise { "The agent runs in with the user's environment minus every CLAUDE* (except CLAUDE_CONFIG_DIR), " + "ANTHROPIC*, CODEX_* (except CODEX_HOME), OP_*, OPENAI_API_KEY, QUEST_GATE_ADMITTED, QUEST_GATE_PRIORITY, " + "QUEST_LAND_BASE, and MBX_SCHEDULER_PRIORITY when QUEST_GATE_PRIORITY=1, so it uses the CLI's stored login rather than the coordinating session's.\n\n" + - "Start and resume first wait until the host and account can carry the agent, as agent admit describes, " + - "before claiming the worktree, and exit 3 with `outcome limited` and no session when the account cannot.\n\n" + + "Start and resume first wait until the host can carry the agent, as agent admit describes, " + + "before claiming the worktree.\n\n" + "Prints `session ` as soon as the id is known, then `outcome finished|failed|limited|logged-out|stopped`, " + "a blank line, and the agent's final reply. Exits 0 when the agent finished, 3 when its account hit a " + "usage limit or ran out of credit, 4 when the CLI is logged out (after printing its login command), 2 for " + @@ -1085,27 +1084,18 @@ export function main(argv: string[], out: Output): number | Promise { ); agent .command("admit") - .summary("Wait until the host and account can carry one more agent") + .summary("Wait until the host can carry one more agent") .description( "Wait until this host can carry one more agent, for a coordinator to run before it starts or resumes " + - "an agent of its own harness, such as a sub-agent; start and resume do the same themselves. The " + - "host has room while quest gate would admit a command at once, counting every queued command and, " + - `as one recent admission each, every agent admitted in the last ${PENDING_MS / 60_000} minutes, since an agent ` + - "adds no gated load until its first build; an agent start or resume runs counts only until it exits. Agents " + + "an agent of its own, such as a sub-agent; start and resume do the same themselves. The host has room " + + "while quest gate would admit a command at once, counting every queued command and, as one recent " + + `admission each, every agent admitted in the last ${PENDING_MS / 60_000} minutes, since an agent adds ` + + "no gated load until its first build; an agent start or resume runs counts only until it exits. Agents " + "already running are never paused.\n\n" + - "The account has room while, in each usage window its harness reports, the usage spent plus what this " + - "agent and the account's agents in flight on this host are expected to spend stays within the window. " + - "Codex reports its windows in its session logs, read from $CODEX_HOME or ~/.codex, and an agent's " + - "expected share is the token-weighted mean share of this host's sessions since the window opened. " + - "Claude Code reports its windows to no command, so it is admitted on the host alone, saying so.\n\n" + - "Exits 0 once admitted, 3 at once when the account cannot cover the agent, naming the reset, 128 plus " + - "the signal number when interrupted, and 1 otherwise.", + "Exits 0 once admitted, 128 plus the signal number when interrupted, and 1 otherwise.", ) - .addOption( - new Option("--harness ", "Agent CLI the agent runs on").choices(HARNESSES).makeOptionMandatory(), - ) - .action((options: { harness: HarnessName }) => { - pending = admitAgent(options.harness, agentDependencies()).then((status) => status ?? 0, agentFailure); + .action(() => { + pending = admitAgent(agentDependencies()).then((status) => status ?? 0, agentFailure); }); agent .command("stop") diff --git a/src/cli/messages.ts b/src/cli/messages.ts index 41076ba1..a870f2a0 100644 --- a/src/cli/messages.ts +++ b/src/cli/messages.ts @@ -8,8 +8,7 @@ import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { discoveryEnv } from "./git"; import { githubRepository } from "./github"; -import { eachLine } from "./lines"; -import { claudeLogs, parseLine } from "./usage"; +import { claudeLogs, eachLine, parseLine } from "./usage"; /** One message a person typed. */ export interface Message { diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 7d1bdfac..d2e24089 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -2,7 +2,7 @@ // the work went by the PR's own record. ccusage reads and prices the sessions; quest ties each to a PR by the // branch its log names, reading only that metadata and never a prompt or reply. import { spawnSync } from "node:child_process"; -import { existsSync, readdirSync } from "node:fs"; +import { closeSync, existsSync, openSync, readdirSync, readSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { type Size, SIZES, handedBack, questBranch } from "../core"; @@ -10,7 +10,6 @@ import { recorded } from "../github/models"; import { type ClaimDependencies, repositoryPath, upstream } from "./claims"; import { discoveryEnv, git } from "./git"; import { fixes } from "./land-fixer"; -import { eachLine } from "./lines"; import { MERGER } from "./land-watch-fix"; import { type GitHub, type ObjectValue, type Repository, githubRepository, link, requireRemote } from "./github"; import { type OnlineReview, readOnlineReviews } from "./online-review"; @@ -211,6 +210,31 @@ export function parseCcusage(text: string): Priced[] { }); } +/** + * Visit each line of the file at `path`, reading it in bounded chunks since logs reach gigabytes. The last line is + * marked incomplete when no newline ends it, as while its session is still writing it. Reading stops once `visit` + * returns false. + */ +export function eachLine(path: string, visit: (line: string, complete: boolean) => boolean | void): void { + const fd = openSync(path, "r"); + try { + const chunk = Buffer.alloc(1 << 22); + let rest = Buffer.alloc(0); + for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) { + const data = rest.length ? Buffer.concat([rest, chunk.subarray(0, read)]) : chunk.subarray(0, read); + let start = 0; + for (let end = data.indexOf(10); end >= 0; end = data.indexOf(10, start)) { + if (end > start && visit(data.toString("utf8", start, end), true) === false) return; + start = end + 1; + } + rest = Buffer.from(data.subarray(start)); + } + if (rest.length) visit(rest.toString("utf8"), false); + } finally { + closeSync(fd); + } +} + /** One JSON record of a log line, or null for an incomplete last line. */ export function parseLine(path: string, line: string, complete: boolean): Record | null { try { diff --git a/test/admission.test.ts b/test/admission.test.ts index dee99bb7..775503db 100644 --- a/test/admission.test.ts +++ b/test/admission.test.ts @@ -1,7 +1,5 @@ -import { mkdirSync, realpathSync, utimesSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; import { expect, test } from "vitest"; -import { type AdmissionDependencies, PENDING_MS, type Window, admit, codexAccount } from "../src/cli/admission"; +import { type AdmissionDependencies, PENDING_MS, admit } from "../src/cli/admission"; import { type Admission, type Host, RECENT_MS, SETTLE_MS, fileHost, processStart } from "../src/cli/gate"; import { tempDir } from "./fixture"; @@ -10,14 +8,13 @@ const signal = new AbortController().signal; const me = { pid: process.pid, start: processStart(process.pid) }; /** A host of 8 CPUs with real ledgers, and load, memory, a clock, and process liveness the test controls. */ -function host(windows: Window[] | null = null) { +function host() { const state = { now: 10 * PENDING_MS, load: 0, memory: Infinity, dead: new Set(), stop: Infinity, - account: "main", }; const sleeps: number[] = []; const notices: string[] = []; @@ -37,15 +34,15 @@ function host(windows: Window[] | null = null) { limits: { load: 8, memory: 4 * GiB }, gate: ledger(tempDir()), agents: ledger(tempDir()), - account: (harness) => windows && { id: `${harness} ${state.account}`, windows }, notice: (text) => notices.push(text), }; const gated = (admissions: Admission[]) => deps.gate.publish(deps.gate.ledger(), admissions); /** Whether one more agent is admitted without waiting. */ - const atOnce = async (harness = "claude", held?: number) => { + const atOnce = async (held?: number) => { state.stop = state.now; try { - return (await admit(harness, deps, signal, held)) === null; + await admit(deps, signal, held); + return true; } catch (error) { if ((error as Error).message !== "still waiting") throw error; return false; @@ -61,7 +58,7 @@ test("a fan-out fills the host's headroom, and each agent frees its share once i expect([await atOnce(), await atOnce(), await atOnce(), await atOnce()]).toEqual([true, true, true, false]); // The fourth waits as the gate's callers do, and starts once the first three no longer count. state.stop = Infinity; - expect(await admit("claude", deps, signal)).toBeNull(); + await admit(deps, signal); expect(sleeps.reduce((sum, ms) => sum + ms, 0)).toBe(PENDING_MS); expect(new Set(sleeps)).toEqual(new Set([SETTLE_MS])); // Each waiting call names what it waits for once, however long it waits. @@ -109,152 +106,11 @@ test("a host loaded by work no gate holds still starts one agent at a time", asy test("an agent whose launcher exited no longer counts, and one still running counts only while it may yet build", async () => { const { state, atOnce } = host(); state.load = 7; - expect([await atOnce("codex", 4242), await atOnce("codex")]).toEqual([true, false]); + expect([await atOnce(4242), await atOnce()]).toEqual([true, false]); state.dead.add(4242); - expect(await atOnce("codex")).toBe(true); - state.now += PENDING_MS; - expect([await atOnce("codex", 4243), await atOnce("codex")]).toEqual([true, false]); + expect(await atOnce()).toBe(true); state.now += PENDING_MS; - expect(await atOnce("codex")).toBe(true); -}); - -test("the account admits an agent only while its windows cover it and the agents in flight", async () => { - const resets = 20 * PENDING_MS; - const { state, deps, atOnce, notices } = host([ - { used: 70, resets, expected: 10 }, - { used: 99, resets: 5 * PENDING_MS, expected: 10 }, - ]); - // The second window reset before now, so it holds nothing; 70 + 10 per agent covers three. - expect([await atOnce("codex", 1), await atOnce("codex", 2), await atOnce("codex", 3)]).toEqual([true, true, true]); - expect(await admit("codex", deps, signal)).toEqual({ resets }); - // Another account's agents spend their own windows, on the same harness or another. - state.account = "other"; - expect(await atOnce("codex", 4)).toBe(true); - expect(await atOnce("claude")).toBe(true); - state.account = "main"; - // An agent that ended is no longer in flight, however recently it started. - state.dead.add(2); - expect(await atOnce("codex")).toBe(true); - // One without a launcher counts as in flight only while it counts against the host. - expect(await admit("codex", deps, signal)).toEqual({ resets }); + expect([await atOnce(4243), await atOnce()]).toEqual([true, false]); state.now += PENDING_MS; - expect(await atOnce("codex")).toBe(true); - expect(notices).toEqual([]); -}); - -test("usage spent while an agent waits for the host refuses it, and an account that changes under it fails", async () => { - const resets = 20 * PENDING_MS; - const windows = [{ used: 50, resets, expected: 10 }]; - const { state, deps, sleeps } = host(windows); - state.memory = GiB; - // Other agents spend the account during the second wait, and memory never frees. - deps.agents.sleep = async (ms) => { - sleeps.push(ms); - state.now += ms; - if (sleeps.length === 2) windows[0].used = 95; - }; - expect(await admit("codex", deps, signal)).toEqual({ resets }); - expect(sleeps).toHaveLength(2); - windows[0].used = 50; - deps.agents.sleep = async () => { - state.account = "other"; - }; - await expect(admit("codex", deps, signal)).rejects.toThrow("stopped reporting the usage of the account"); -}); - -test("a harness that reports no usage is admitted on the host alone, saying so", async () => { - const { atOnce, notices } = host(null); - expect(await atOnce("claude")).toBe(true); - expect(notices).toEqual(["claude reports no account usage here, so only the host is checked"]); -}); - -const minute = 60_000; -const now = 1_000_000 * minute; - -/** - * A Codex rollout at `path` under `home`, last written at `modified` ms, holding a `token_count` - * event per `[time, total tokens, rate limits]`, then `tail`. - */ -function rollout(home: string, path: string, modified: number, events: [number, number, unknown][], tail = "") { - const file = join(home, path.includes("/") ? path : `sessions/2026/11/12/rollout-${path}.jsonl`); - mkdirSync(dirname(file), { recursive: true }); - const lines = events.map(([at, tokens, limits]) => - JSON.stringify({ - timestamp: new Date(at).toISOString(), - type: "event_msg", - payload: { - type: "token_count", - info: { total_token_usage: { total_tokens: tokens } }, - rate_limits: limits, - }, - }), - ); - writeFileSync(file, `{"type":"session_meta"}\n${lines.join("\n")}\n${tail}`); - utimesSync(file, modified / 1000, modified / 1000); -} - -const week = { used_percent: 40, window_minutes: 10_080, resets_at: (now + 1000 * minute) / 1000 }; -const hours = { used_percent: 10, window_minutes: 300, resets_at: (now + 100 * minute) / 1000 }; -const old = { primary: { ...week, used_percent: 5 }, secondary: null }; - -test("Codex's windows come from its newest report, with an agent's share from this host's sessions", () => { - const home = tempDir(); - // Written before the 5-hour window opened, inside the week's. - rollout(home, "a", now - 400 * minute, [[now - 400 * minute, 300, old]]); - // An archived session still spent the window, and one seen in both trees counts once. - rollout(home, "archived_sessions/rollout-b.jsonl", now - 50 * minute, [[now - 50 * minute, 100, old]]); - rollout(home, "sessions/2026/11/13/rollout-b.jsonl", now - 50 * minute, [[now - 50 * minute, 100, old]]); - // The newest report's rollout has a last line still being written, so its last whole one counts. - rollout( - home, - "c", - now - 2 * minute, - [[now - 2 * minute, 100, { primary: week, secondary: hours }]], - '{"type":"event_msg","payload":{"type":"token_c', - ); - // A rollout written since, by a tool's output, holds only an older report. - rollout(home, "d", now - minute, [[now - 30 * minute, 0, old]]); - const { id, windows } = codexAccount(home, now)!; - expect(id).toBe(`codex ${realpathSync(home)}`); - // Sessions of 300, 100, and 100 tokens: a token's session holds (9 + 1 + 1) / 25 of the window's spend. - expect(windows).toEqual([ - { used: 40, resets: now + 1000 * minute, expected: (40 * 11) / 25 }, - { used: 10, resets: now + 100 * minute, expected: 10 / 2 }, - ]); - // A window that reset since the report is unknown, not spent. - expect(codexAccount(home, now + 200 * minute)!.windows).toEqual([expect.objectContaining({ used: 40 })]); -}); - -test("a session resumed across a window's reset counts only its tokens since", () => { - const home = tempDir(); - const opened = now + 100 * minute - 300 * minute; - // 900 of its 1000 tokens were spent before the 5-hour window opened. - rollout(home, "1970-06-01T00-00-00-a", now - minute, [ - [opened - 500 * minute, 400, old], - [opened - minute, 900, old], - [now - minute, 1000, { primary: hours, secondary: null }], - ]); - rollout(home, "b", now - 10 * minute, [[now - 10 * minute, 100, old]]); - // Two sessions of 100 tokens each in the window: half of its 10 points apiece. - expect(codexAccount(home, now)!.windows).toEqual([{ used: 10, resets: now + 100 * minute, expected: 5 }]); -}); - -test("two Codex homes are one account only when their logins name the same one", () => { - const [first, second] = [tempDir(), tempDir()]; - for (const home of [first, second]) rollout(home, "a", now, [[now, 1, { primary: week, secondary: null }]]); - expect(codexAccount(first, now)!.id).not.toBe(codexAccount(second, now)!.id); - for (const home of [first, second]) writeFileSync(join(home, "auth.json"), '{"tokens":{"account_id":"acct"}}'); - expect([codexAccount(first, now)!.id, codexAccount(second, now)!.id]).toEqual(["codex acct", "codex acct"]); - // An API key's login names no account. - writeFileSync(join(second, "auth.json"), '{"OPENAI_API_KEY":"sk","tokens":null}'); - expect(codexAccount(second, now)!.id).toBe(`codex ${realpathSync(second)}`); -}); - -test("Codex without a usage report gives no signal, and an unreadable one fails", () => { - const home = tempDir(); - expect(codexAccount(home, 0)).toBeNull(); - rollout(home, "a", 1000, [[1000, 100, null]]); - expect(codexAccount(home, 0)).toBeNull(); - rollout(home, "b", 2000, [[2000, 100, { primary: { used_percent: "40%" }, secondary: null }]]); - expect(() => codexAccount(home, 0)).toThrow("Codex reported a primary usage window quest cannot read"); + expect(await atOnce()).toBe(true); }); diff --git a/test/agent.test.ts b/test/agent.test.ts index 1af56b47..30e26637 100644 --- a/test/agent.test.ts +++ b/test/agent.test.ts @@ -24,7 +24,7 @@ import { Worktrees } from "../src/cli/worktree"; // Launches here are admitted at once, whatever the host running the tests carries. beforeEach(() => { - vi.spyOn(admission, "admit").mockResolvedValue(null); + vi.spyOn(admission, "admit").mockResolvedValue(); }); afterEach(() => { @@ -371,42 +371,27 @@ test("an account at its usage limit exits 3, and the session resumes once it res expect(call.stdin).toBe("continue"); }); -test("a launch waits for admission before it claims the worktree, and an account that cannot cover it exits 3", async () => { +test("a launch waits for admission before it claims the worktree", async () => { const s = setup(); - const admit = vi.spyOn(admission, "admit").mockImplementation(async (harness, deps, signal, held) => { - expect([harness, held, signal.aborted]).toEqual(["claude", process.pid, false]); + vi.spyOn(admission, "admit").mockImplementation(async (deps, signal, held) => { + expect([held, signal.aborted]).toEqual([process.pid, false]); // Nothing holds the worktree while the launch waits. const state = join(s.runtime, "quest", "agent"); expect(readdirSync(join(state, readdirSync(state)[0], "agents"))).toEqual([]); deps.notice("waiting for the load average to fall before starting an agent"); - return null; }); const started = await quest("agent", "start", ...launch("claude"), s.worktree, "go"); expect(started).toMatchObject({ code: 0, stderr: "quest: waiting for the load average to fall before starting an agent\n", }); - - admit.mockResolvedValue({ resets: Date.UTC(2026, 10, 14, 15) }); - const session = /^session (\S+)$/m.exec(started.stdout)![1]; - const resumed = await quest("agent", "resume", ...launch("claude"), s.worktree, session, "continue"); - const refusal = "quest: claude's reported usage cannot cover another agent; resets 2026-11-14T15:00:00.000Z\n"; - expect(resumed).toEqual({ code: 3, stdout: "outcome limited\n", stderr: refusal }); - expect(s.calls()).toHaveLength(1); }); -test("agent admit exits 0 once admitted and 3 when the account cannot cover an agent of its own", async () => { - const admit = vi.spyOn(admission, "admit").mockResolvedValue(null); - expect(await quest("agent", "admit", "--harness", "codex")).toEqual({ code: 0, stdout: "", stderr: "" }); +test("agent admit exits 0 once admitted", async () => { + const admit = vi.spyOn(admission, "admit").mockResolvedValue(); + expect(await quest("agent", "admit")).toEqual({ code: 0, stdout: "", stderr: "" }); // No process runs a coordinator's own sub-agent, so nothing holds its admission. - expect(admit.mock.calls[0][0]).toBe("codex"); - expect(admit.mock.calls[0][3]).toBeUndefined(); - admit.mockResolvedValue({ resets: 0 }); - expect(await quest("agent", "admit", "--harness", "codex")).toMatchObject({ - code: 3, - stderr: "quest: codex's reported usage cannot cover another agent; resets 1970-01-01T00:00:00.000Z\n", - }); - expect((await quest("agent", "admit")).code).toBe(2); + expect(admit.mock.calls[0][2]).toBeUndefined(); }); test("a spent free usage allowance exits 3 as limited", async () => { From 8a38ffe02b6846312c36341cdfc3dd9aa693df5d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 09:00:28 -0800 Subject: [PATCH 6/6] fix(agent): restart a future-dated agent entry from now so it still expires Co-Authored-By: Claude Opus 5.5 --- src/cli/admission.ts | 12 +++++++++--- test/admission.test.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/cli/admission.ts b/src/cli/admission.ts index c1871db4..4fc0d169 100644 --- a/src/cli/admission.ts +++ b/src/cli/admission.ts @@ -56,15 +56,21 @@ export async function admit(deps: AdmissionDependencies, signal: AbortSignal, he signal.throwIfAborted(); const ledger = deps.agents.ledger(); const now = deps.agents.now(); + // A clock stepped backward dates an entry in the future; restart it from now, so it still expires. + if (ledger.admissions.some((entry) => entry.at > now)) { + deps.agents.publish( + ledger, + ledger.admissions.map((entry) => (entry.at > now ? { ...entry, at: now } : entry)), + ); + continue; + } const pending = ledger.admissions.filter( (entry) => entry.at > now - PENDING_MS && (entry.pid === undefined || deps.agents.running(entry)), ); const reason = room(deps.limits, deps.gate, pending.length); if (reason === null) { - // A clock stepped backward dates an entry in the future; count it from now instead. - const kept = pending.map((entry) => (entry.at > now ? { ...entry, at: now } : entry)); const entry: Admission = { id: randomUUID(), at: now, ...holder }; - if (deps.agents.publish(ledger, [...kept, entry])) return; + if (deps.agents.publish(ledger, [...pending, entry])) return; continue; } if (!shown.has(reason)) { diff --git a/test/admission.test.ts b/test/admission.test.ts index 775503db..c09d0252 100644 --- a/test/admission.test.ts +++ b/test/admission.test.ts @@ -114,3 +114,11 @@ test("an agent whose launcher exited no longer counts, and one still running cou state.now += PENDING_MS; expect(await atOnce()).toBe(true); }); + +test("an agent dated in the future by a clock stepped backward counts from now, and still expires", async () => { + const { state, deps, sleeps } = host(); + state.load = 7; + deps.agents.publish(deps.agents.ledger(), [{ id: "ahead", at: state.now + 12 * PENDING_MS }]); + await admit(deps, signal); + expect(sleeps.reduce((sum, ms) => sum + ms, 0)).toBe(PENDING_MS); +});