From 2bd7cd671d61af216859901dc4c122848cae1674 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 17 Sep 2026 21:53:56 +0700 Subject: [PATCH 1/2] fix(queen): nothing in dispatch asked the container whether another bee fits, so ask before every start The worker ceiling is a number and the credential list is a number, and the one thing that actually decides how many bees run at once - the container - was never asked. Measured on 2026-09-17 from the Railway metrics: the container is limited to 24 GB and held 14.7 GB with sixteen bees running, about 0.85 GB a bee over 1 GB idle; the volume is 50 GB with 29 to 31 in use and 35.8 at the day's peak. The same day the ceiling stopped being a constant and the credential list reached 36 lanes. A full swarm would ask for about 31 GB of 24, and the kernel answers that by killing the container with every running bee in it, each holding the only copy of its unfinished turn. queen-resources.ts is whether one more bee fits. Before every start it reads what the container holds that the kernel cannot give back - memory.current minus the file cache, both lists, minus reclaimable slab - against memory.max, and the free space on the workspace volume. cgroup v2, then v1, then /proc/meminfo. Nothing is cached, so the line moves with the container and with the operator's variables and no restart is needed. Memory refuses when used + (young + 1) x perBee > limit - min(10%, 8 GB); disk when free < perBeeDisk + min(10%, 10 GB). Memory first, one cause reported. Unknown is a refusal; a machine that is not Linux has no container to protect. Young bees: the tick starts bees back to back and a bee takes minutes to allocate what it will hold, so every start inside the warm-up reserves a bee's memory the reading cannot show yet. The reservation is keyed by the turn and ends in closeDispatch, because /chat answers 200 and streams a provider refusal afterwards: kept as timestamps, a provider outage reserved ten GB for bees that were already dead. The gate sits in dispatchBee after the key refusals and before the worktree is cut. Disk gets one reap before it refuses, bounded at the guard's own line rather than the collector's 55%. A container refusal is not booked against the issue: it says nothing about the issue and is how a memory-bound round ends. The reaper no longer takes the tree of a running bee. reapWorktrees takes a protect set and prepareWorktree's own reap gets it too - that one fires at 80% used, earlier than the guard refuses, and on a scratch repository it removed two trees registered as running. The gate's reap also protects the branch being dispatched: a re-dispatch could reap its own tree and then cut it afresh with -B, dropping the last attempt's unpushed commit. An unreadable registry reaps nothing instead of protecting nobody. The round report says why. /queen/needs-you serves the headline to a browser and promises no path, so the headline gets a label from a closed vocabulary (no room in the container: memory | disk) and the body gets the guard's summary, carried as a field and never cut out of prose that names a path. Checked from inside a Railway container rather than assumed: memory.max reads 24000000000, memory.current and memory.stat are readable, there is no cgroup v1, and /proc/meminfo describes the host (338 GB). So a stated TRIOS_QUEEN_MEMORY_LIMIT_MB can only lower the line - the kernel's limit wins - and the host is never held against a stated container limit. Numbers are printed in decimal GB, the unit the platform sells and the cgroup file uses. Admission control only: a bee admitted and then grown far past its reserve can still take the container down. It caps nothing by count; the Swift copies, the closed capacity breakdown (#1308) and /queen/status are untouched. Two review rounds found and these tests now pin: the unprotected reap, the re-dispatch reaping its own tree, dead starts holding reservations, a stated limit above the kernel's, active cache counted as used, a missing checkout read as a full disk. Each new behavioural test was checked by breaking the line it guards. Co-Authored-By: Claude Fable 5.1 --- .../server/src/api/services/queen-dispatch.ts | 274 +++++++- .../src/api/services/queen-report-lines.ts | 49 ++ .../src/api/services/queen-resources.ts | 575 +++++++++++++++++ .../server/src/api/services/queen-tick.ts | 19 +- .../server/tests/api/queen-dispatch.test.ts | 487 +++++++++++++- .../tests/api/queen-report-lines.test.ts | 137 +++- .../server/tests/api/queen-resources.test.ts | 595 ++++++++++++++++++ .../apps/server/tests/api/queen-round.test.ts | 67 +- .../server/tests/api/queen-volume-gc.test.ts | 222 ++++++- 9 files changed, 2392 insertions(+), 33 deletions(-) create mode 100644 trios/agent-server/apps/server/src/api/services/queen-resources.ts create mode 100644 trios/agent-server/apps/server/tests/api/queen-resources.test.ts diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 3423171b33..6bf018529e 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -29,6 +29,19 @@ import { statSync } from 'node:fs' import type { Pool } from 'pg' import { logger } from '../../lib/logger' import { shellArgv } from '../../tools/filesystem/bash' +import { + describeReading, + diskLineUsedPercent, + judgeBeeRoom, + type MemoryReading, + noteBeeEnded, + noteBeeStarted, + readContainerMemory, + resourceSettings, + type VolumeSpace, + volumeSpace, + youngBeeCount, +} from './queen-resources' import { workerSystemPrompt } from './queen-tick' /** @@ -1673,12 +1686,22 @@ export async function reapWorktrees( low?: number keepNewest?: number volumeUsed?: (dir: string) => number | null + /** + * Branch names (`queen-`) of bees that are running right now. + * + * "The newest few are likely to be running" was true while four bees ran. + * With twenty, the seventh-newest tree belongs to a bee too, and a bee that + * has read a lot and written nothing yet has a CLEAN tree - exactly what + * this function removes. Its work would vanish under it mid-turn. + */ + protect?: ReadonlySet } = {}, ): Promise<{ before: number | null after: number | null removed: string[] keptDirty: string[] + keptRunning: string[] refused: string[] }> { const high = opts.high ?? Number(process.env.QUEEN_VOLUME_HIGH ?? 80) @@ -1693,6 +1716,7 @@ export async function reapWorktrees( after: before, removed: [] as string[], keptDirty: [] as string[], + keptRunning: [] as string[], refused: [] as string[], } // Unknown is not room. Below the mark is not an emergency. @@ -1722,6 +1746,10 @@ export async function reapWorktrees( const now = measure(root) if (now !== null && now <= low) break + if (opts.protect?.has(c.path.slice(c.path.lastIndexOf('/') + 1))) { + result.keptRunning.push(c.path) + continue + } const dirty = await run('git', ['status', '--porcelain'], c.path, 60_000) // Unreadable is not clean. A tree whose state cannot be read might hold the // only copy of a turn's work. @@ -1863,6 +1891,18 @@ export async function prepareWorktree( // filesystem is a guard no test can pin. So the measurement is a dependency // with a real default, like the pool and the clock everywhere else here. volumeUsed?: (dir: string) => number | null + /** + * The branches of bees running right now, asked for only when a reap is + * about to happen. + * + * Without it this reap took the clean tree of a running bee. It fires at + * QUEEN_VOLUME_HIGH - 80% used - which on a 50 GB volume is EARLIER than the + * container guard refuses (7 GB free is 86%), so between the two marks the + * guard saw room, reaped nothing, and this call reaped unprotected. + * Reproduced on a scratch repository before it was fixed: ten clean trees, + * two of them registered as running, both gone. + */ + running?: () => Promise | null> } = {}, ): Promise<{ ok: boolean; path: string; detail: string }> { const measure = deps.volumeUsed ?? volumeUsedPercent @@ -1972,15 +2012,24 @@ export async function prepareWorktree( const used = measure(root) const highMark = Number(process.env.QUEEN_VOLUME_HIGH ?? 80) if (used !== null && used >= highMark) { - const gc = await reapWorktrees({ volumeUsed: measure }) - logger.warn('Queen reaped worktrees before cutting a new one', { - before: gc.before, - after: gc.after, - removed: gc.removed.length, - keptDirty: gc.keptDirty.length, - refused: gc.refused.length, - }) - const still = gc.after + // Null is "the registry could not be read": nothing is reaped then, and the + // refusal below still stands between a full volume and a half-cut tree. + const protect = deps.running ? await deps.running() : undefined + const gc = + protect === null + ? null + : await reapWorktrees({ volumeUsed: measure, protect }) + if (gc) { + logger.warn('Queen reaped worktrees before cutting a new one', { + before: gc.before, + after: gc.after, + removed: gc.removed.length, + keptDirty: gc.keptDirty.length, + keptRunning: gc.keptRunning.length, + refused: gc.refused.length, + }) + } + const still = gc ? gc.after : used if (still !== null && still >= 95) { // REFUSE, and say what is true. Dying at `git worktree add` reports a git // error for a disk problem, and every reader of that message has looked in @@ -1989,10 +2038,13 @@ export async function prepareWorktree( return { ok: false, path, - detail: - `volume ${still}% full after reaping ${gc.removed.length} worktree(s); ` + - `${gc.keptDirty.length} held uncommitted work and were kept. ` + - 'Not cutting a worktree that would fail part-way', + detail: gc + ? `volume ${still}% full after reaping ${gc.removed.length} worktree(s); ` + + `${gc.keptDirty.length} held uncommitted work and ` + + `${gc.keptRunning.length} belong to running bees and were kept. ` + + 'Not cutting a worktree that would fail part-way' + : `volume ${still}% full and nothing was reaped, because the running ` + + 'bees could not be listed. Not cutting a worktree that would fail part-way', } } } @@ -2617,6 +2669,9 @@ export async function closeDispatch( outcome: string, tokens?: TokenUsage, ): Promise { + // The stream is over, so what the container guard reserved for this bee while + // it was young is over too - whatever the database does below. + noteBeeEnded(conversationId) // Whether the row reads finished on the database when this returns. That is // the ONLY condition under which the slot may be announced as free: a signal // about a row that still says `running` wakes a round that sees the bee as @@ -2792,6 +2847,13 @@ export interface DispatchOutcome { conversationId?: string /** Which provider key this bee took, so the next one takes a different one. */ keyIndex?: number + /** + * Set only when the CONTAINER refused the start. The report reads it to tell + * this refusal from every other one, because this is the only one whose + * sentence it may quote outside the "Refused #N" line - and it quotes + * `summary`, never a piece cut out of `detail`. + */ + room?: { resource: 'memory' | 'disk'; summary: string } } /** @@ -2814,6 +2876,8 @@ export async function dispatchBee( */ criteria: string[] = [], criteriaSource = 'none', + /** Measurements, injectable so a test can put the container in any state. */ + deps: BeeRoomDeps = {}, ): Promise { const branch = `queen-${issue}` @@ -2856,7 +2920,37 @@ export async function dispatchBee( return { started: false, issue, branch, detail } } - const worktree = await prepareWorktree(issue) + // A key is free and a provider answers. The remaining question is the one + // nothing used to ask: can the CONTAINER carry another bee? Asked here, before + // a worktree is cut, so a refusal costs a measurement and nothing else. The + // tick loop breaks on a refused dispatch and the next round fires when a bee + // finishes - which is exactly when memory comes back. + const noRoom = await beeRoomRefusal(pool, branch, deps) + if (noRoom) { + logger.warn( + 'Queen tick chose an issue but the container cannot carry another bee', + { issue, resource: noRoom.resource, detail: noRoom.detail }, + ) + // NOT RECORDED AGAINST THE ISSUE, unlike every refusal above. Those say + // something about the dispatch; this says something about the container, + // and once the swarm is memory-bound it is how nearly every round ends. + // `recordDispatch` would archive a history row and rewrite the issue's live + // row each time - wiping the last attempt's conversation, tokens and + // outcome for a reason that has nothing to do with the issue. The warning + // above and the round report, one row per round already, carry it. + return { + started: false, + issue, + branch, + detail: noRoom.detail, + room: { resource: noRoom.resource, summary: noRoom.summary }, + } + } + + const worktree = await prepareWorktree(issue, { + running: () => runningBeeBranches(pool), + volumeUsed: deps.volumeUsed, + }) if (!worktree.ok) { await recordDispatch( pool, @@ -2925,6 +3019,9 @@ export async function dispatchBee( chosen.provider, chosen.model, ) + // A bee that just started has not allocated what it will hold. The next + // dispatch of this round must not read the container as empty because of it. + if (turn.ok) noteBeeStarted(conversationId, (deps.now ?? Date.now)()) // ONLY NOW may the stream be read. Everything that reads the bee's output // eventually writes to the row above, and a writer that can outrun the row's // creation is a writer that silently updates nothing. @@ -2940,6 +3037,155 @@ export async function dispatchBee( } } +export interface BeeRoomDeps { + memory?: () => MemoryReading + volume?: (dir: string) => VolumeSpace | null + reap?: typeof reapWorktrees + now?: () => number + /** Handed to `prepareWorktree`, whose own volume check reads the real disk. */ + volumeUsed?: (dir: string) => number | null +} + +/** + * The sentence that refuses a start because the container has no room, or null. + * + * Disk gets one chance to make room before it refuses, because it is the one + * resource this server can free itself: the reaper only ever ran inside + * `prepareWorktree`, which is AFTER this gate, so a gate that refused on a full + * volume would have stopped the only code that empties it. The reap protects + * the trees of running bees - see `reapWorktrees`. Memory gets no such chance: + * nothing here can free it except a bee finishing. + */ +async function beeRoomRefusal( + pool: Pool, + branch: string, + deps: BeeRoomDeps, +): Promise<{ + resource: 'memory' | 'disk' + summary: string + detail: string +} | null> { + const settings = resourceSettings() + if (!settings.guardOn) { + sayOncePerChange( + 'Queen resource guard is OFF by TRIOS_QUEEN_RESOURCE_GUARD; dispatching without checking memory or disk', + {}, + true, + ) + return null + } + const root = workspaceRoot() + const now = (deps.now ?? Date.now)() + const judge = (reaped?: { + removed: number + keptDirty: number + keptRunning: number + }) => { + const memory = (deps.memory ?? readContainerMemory)() + const volume = (deps.volume ?? volumeSpace)(root) + // Only the REAL measurement can mistake a missing checkout for a sick disk; + // an injected one answers for itself. + const checkoutMissing = !deps.volume && !pathExists(root) + const notes = [ + ...settings.notes, + ...(memory.kind === 'measured' && memory.note ? [memory.note] : []), + ] + // What it reads, once - so the first thing an operator checks after a + // deploy (is it looking at the container or at the host?) is in the log + // before anything is ever refused. + sayOncePerChange( + 'Queen resource guard is on', + { reads: describeReading(memory), volume: root, ignored: notes }, + notes.length > 0, + ) + const room = judgeBeeRoom({ + memory, + volume, + youngBees: youngBeeCount(now, settings.warmupSeconds), + settings, + volumeDir: root, + reaped, + checkoutMissing, + }) + return { room, volume } + } + const first = judge() + if (first.room.ok) return null + if (first.room.resource !== 'disk' || first.volume === null) return first.room + + // WHO IS RUNNING MUST BE KNOWN, or nothing is reaped. An unreadable registry + // used to mean "protect nobody", which is the unprotected reap this change + // exists to end; and skipping a reap refuses nothing the disk had not already + // refused. The next round asks again. + const running = await runningBeeBranches(pool) + if (running === null) return first.room + const gc = await (deps.reap ?? reapWorktrees)({ + // ...and the tree of the issue being dispatched, which is by definition not + // running. Without it a re-dispatch could reap its own tree here, and + // `prepareWorktree` would then cut it afresh with `-B`: the previous + // attempt's unpushed commits dropped off the branch. Reproduced on a + // scratch repository before this line existed. + protect: new Set([...running, branch]), + // `high: 0`: the gate has already established that the volume is short for + // THIS container, whatever percentage that happens to be. `low`: and it + // stops where the rule is satisfied, not at the collector's own low mark. + high: 0, + low: diskLineUsedPercent(first.volume, settings), + }) + const second = judge({ + removed: gc.removed.length, + keptDirty: gc.keptDirty.length, + keptRunning: gc.keptRunning.length, + }) + return second.room.ok ? null : second.room +} + +/** + * Said when it changes, not on every dispatch. The guard runs before every + * start; a line per start about a setting that has not moved is how a log + * teaches its reader to skip the line that matters. + */ +let guardLastSaid = '' +function sayOncePerChange( + message: string, + fields: Record, + warn: boolean, +): void { + const said = `${message} ${JSON.stringify(fields)}` + if (said === guardLastSaid) return + guardLastSaid = said + if (warn) logger.warn(message, fields) + else logger.info(message, fields) +} + +/** + * The branch names (`queen-`) of the bees the registry says are running, + * or NULL when the registry cannot be read. + * + * A worktree directory is named after its branch, so this is the set a reap + * must not touch however old and clean a tree looks. Null and not an empty set: + * "nobody is running" and "I could not find out" are different answers, and a + * caller that reaps on the second one removes the trees of running bees. + */ +export async function runningBeeBranches( + pool: Pool, +): Promise | null> { + try { + const rows = await pool.query( + `SELECT issue FROM queen_dispatch + WHERE started = true AND finished_at IS NULL`, + ) + return new Set( + (rows.rows ?? []).map((row: { issue: unknown }) => `queen-${row.issue}`), + ) + } catch (error) { + logger.warn('Queen could not list running bees, so nothing is reaped', { + error: error instanceof Error ? error.message : String(error), + }) + return null + } +} + export async function recordDispatch( pool: Pool, issue: number, diff --git a/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts b/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts index 74cfdcb44f..b2718ea75f 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts @@ -26,6 +26,9 @@ export interface DispatchReportOutcome { started?: boolean issue?: number detail?: string + /** Set only when the CONTAINER refused the start: which resource had no + * room, and the guard's own one-sentence account of it. */ + room?: { resource: 'memory' | 'disk'; summary?: string } } /** @@ -111,6 +114,52 @@ export function refusedLines( }) } +/** + * Why a round that `queend` allowed still started nothing, when the reason is + * the container. + * + * `queend` refuses for the policy's reasons - the worker limit, the budget, + * nothing to choose - and those reach the headline. A refusal made by the + * container guard in DISPATCH did not: `queend` had said yes, so the report + * read "Started nothing. No reason given." under the headline "nothing to do", + * while the actual sentence sat one line lower where a headline reader never + * looks. + * + * TWO STRINGS, BECAUSE THEY GO TO TWO PLACES. The headline is served to a + * browser by `/queen/needs-you`, whose contract is no path, no branch and no + * connection detail - so it gets a label from a closed vocabulary and never + * the sentence, which names the workspace path. The body is the operator's, + * and it gets the guard's own SUMMARY: one sentence that carries every number + * and is written to fit (see `judgeBeeRoom`). Only outcomes the guard marked + * with `room` are read, so git output and provider bodies - which are also + * details of dispatches that did not start - can reach neither string. + * + * The summary is a FIELD and is never cut out of `detail`: the prose names a + * path, and a path may hold ". " - "/Volumes/Ext. Drive" ended the sentence + * after "Ext" while this function still split on it. + * + * It comes without a closing period because `nothingStartedLine` adds exactly + * one. One that is too long anyway is cut at a word and closed with two dots, + * which that period makes the ellipsis this file uses. + */ +export function containerRefusal( + outcomes: ReadonlyArray, +): { headline: string; sentence: string } | null { + const refused = outcomes.find( + (outcome) => outcome.started !== true && outcome.room !== undefined, + ) + if (!refused?.room) return null + const headline = `no room in the container: ${refused.room.resource}` + const summary = String(refused.room.summary ?? '') + .trim() + .replace(/[.!?\s]+$/, '') + const sentence = + summary.length <= DETAIL_SHOWN + ? summary + : `${summary.slice(0, DETAIL_SHOWN - 3).replace(/\s+\S*$/, '')}..` + return { headline, sentence: sentence || headline } +} + /** * The headline: escalations first, because a decision waiting on a human * outranks everything else; then the bees actually working, counted from the diff --git a/trios/agent-server/apps/server/src/api/services/queen-resources.ts b/trios/agent-server/apps/server/src/api/services/queen-resources.ts new file mode 100644 index 0000000000..2f0e3af4ca --- /dev/null +++ b/trios/agent-server/apps/server/src/api/services/queen-resources.ts @@ -0,0 +1,575 @@ +/** + * Whether the container can carry one more bee. + * + * THE MEASUREMENT, 2026-09-17. The container is limited to 24 GB. With sixteen + * bees running it held 14.7 GB, idle it holds one to three: about 0.85 GB per + * running bee. The same day the worker ceiling stopped being a constant and the + * credential list grew to thirty-four lanes. A full swarm would therefore have + * asked for about thirty GB of a twenty-four GB container, and the kernel + * answers that question by killing the container - every running bee with it, + * each holding the only copy of its unfinished turn. The volume is the second + * way to die: 50 GB, 31 in use, 36 at the day's peak. + * + * Nothing in dispatch looked at memory at all. The ceiling was a number, the + * credentials were a number, and the one thing that actually decides how many + * bees fit - the container - was not asked. + * + * So it is asked, before every start, and it is asked AGAIN every time: every + * variable and every reading below is taken on the call, never cached, so the + * line moves with the container and with the operator's variables and no + * restart is ever needed for the guard to notice either. + * + * WHAT THIS IS NOT. It is admission control. A bee that was admitted and then + * grows far past its reserve can still take the container down; the headroom is + * the only buffer against that. And it caps nothing by count - the policy and + * the credentials still decide how many bees MAY run; this only decides + * whether the next one FITS. + */ +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' + +export type MemoryReading = + | { + kind: 'measured' + usedBytes: number + limitBytes: number + /** Where the use was read: 'cgroup v2', 'cgroup v1' or '/proc/meminfo'. */ + source: string + /** Where the limit came from: 'cgroup', 'host MemTotal' or the variable. */ + limitSource: string + /** Said once in the log: a stated limit the kernel's own limit overrules. */ + note?: string + } + /** Not Linux: a development or test machine has nothing to read. */ + | { kind: 'unsupported'; platform: string } + /** Linux, and nothing usable. Unknown is not room. */ + | { kind: 'unknown'; tried: string[]; why?: string } + +export interface VolumeSpace { + totalBytes: number + freeBytes: number +} + +export interface ResourceSettings { + guardOn: boolean + beeMemoryBytes: number + memoryHeadroomPercent: number + warmupSeconds: number + beeDiskBytes: number + diskHeadroomPercent: number + memoryLimitBytes: number | null + /** One sentence per variable that was set to something unusable. */ + notes: string[] +} + +export type BeeRoom = + | { ok: true } + | { + ok: false + resource: 'memory' | 'disk' + /** + * The first sentence of `detail`, without its period: every number, no + * variable name, under 200 characters. A field rather than something a + * reader cuts out of the prose, because the prose names the workspace + * path and a path may contain ". " - "/Volumes/Ext. Drive" ended the + * sentence after "Ext" when the report split on it. + */ + summary: string + detail: string + } + +const MB = 1024 * 1024 +/** + * DECIMAL, for the headroom caps and for every number printed. Railway sells + * "24 GB" and its cgroup file says 24000000000; printed in binary units that is + * "22.4 GB", which reads as a wrong limit and invites an operator to "correct" + * it with TRIOS_QUEEN_MEMORY_LIMIT_MB. The `_MB` variables stay binary, as + * every memory flag an operator has met is. + */ +const GB = 1_000_000_000 +/** + * The headroom is a share of the limit, and a share of something huge is huge: + * ten percent of a developer's terabyte disk is a hundred GB "kept free", and + * the guard would refuse every local dispatch on a laptop with ninety GB to + * spare. The share protects a SMALL container; past these sizes it stops + * growing. + */ +const MEMORY_HEADROOM_CAP = 8 * GB +const DISK_HEADROOM_CAP = 10 * GB +/** cgroup v1 writes "no limit" as a number near 2^63. */ +const V1_UNLIMITED = 2 ** 60 + +function gb(bytes: number): string { + return (bytes / GB).toFixed(1) +} + +function readFileOrNull(path: string): string | null { + try { + return readFileSync(path, 'utf8') + } catch { + return null + } +} + +function statValue(stat: string | null, key: string): number { + if (!stat) return 0 + const match = new RegExp(`^${key}\\s+(\\d+)$`, 'm').exec(stat) + return match ? Number(match[1]) : 0 +} + +function meminfoBytes(meminfo: string | null, key: string): number | null { + if (!meminfo) return null + const match = new RegExp(`^${key}:\\s+(\\d+)\\s*kB`, 'm').exec(meminfo) + return match ? Number(match[1]) * 1024 : null +} + +/** + * Read on every call. A value that cannot be used is replaced by its default + * AND SAID SO: a variable that is silently ignored is the zero-length-key trap + * again - it looks configured in the editor and changes nothing. + */ +export function resourceSettings( + env: NodeJS.ProcessEnv = process.env, +): ResourceSettings { + const notes: string[] = [] + const bounded = ( + name: string, + fallback: number, + low: number, + high: number, + ): number => { + const raw = env[name]?.trim() + if (!raw) return fallback + const parsed = Number(raw) + if (!Number.isInteger(parsed) || parsed < low || parsed > high) { + notes.push( + `${name}=${raw.slice(0, 24)} is not an integer from ${low} to ${high}; using ${fallback}`, + ) + return fallback + } + return parsed + } + const guardRaw = env.TRIOS_QUEEN_RESOURCE_GUARD?.trim().toLowerCase() + if (guardRaw && guardRaw !== 'on' && guardRaw !== 'off') { + notes.push( + `TRIOS_QUEEN_RESOURCE_GUARD=${guardRaw.slice(0, 24)} is neither on nor off; the guard stays on`, + ) + } + // Not `bounded`: this one has no default to fall back TO. Unset, empty and 0 + // all mean "read the limit from the container", and an unusable value must + // say THAT - "using 0" would tell an operator the container has no memory. + const limitRaw = env.TRIOS_QUEEN_MEMORY_LIMIT_MB?.trim() + let limitMb = 0 + if (limitRaw && limitRaw !== '0') { + const parsed = Number(limitRaw) + if (Number.isInteger(parsed) && parsed >= 512 && parsed <= 4_194_304) { + limitMb = parsed + } else { + notes.push( + `TRIOS_QUEEN_MEMORY_LIMIT_MB=${limitRaw.slice(0, 24)} is not an integer from 512 to 4194304; ` + + 'using the limit the container reports', + ) + } + } + return { + guardOn: guardRaw !== 'off', + beeMemoryBytes: + bounded('TRIOS_QUEEN_BEE_MEMORY_MB', 1024, 128, 65_536) * MB, + memoryHeadroomPercent: bounded( + 'TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT', + 10, + 0, + 50, + ), + warmupSeconds: bounded('TRIOS_QUEEN_BEE_WARMUP_SECONDS', 600, 0, 3600), + beeDiskBytes: bounded('TRIOS_QUEEN_BEE_DISK_MB', 2048, 256, 1_048_576) * MB, + diskHeadroomPercent: bounded( + 'TRIOS_QUEEN_DISK_HEADROOM_PERCENT', + 10, + 0, + 50, + ), + memoryLimitBytes: limitMb > 0 ? limitMb * MB : null, + notes, + } +} + +/** + * What the container holds that the kernel CANNOT give back, against its limit. + * + * Not `memory.current`: that counts page cache, and a server that clones + * repositories and links node_modules fills the cache constantly. And not the + * kubelet's working set either (`current - inactive_file`), which was the first + * version: it still counts `active_file` and reclaimable slab, and those come + * back only under pressure AT the limit - which this guard refuses before ever + * reaching. On a classic-LRU kernel a page read twice is promoted to active and + * nothing demotes it without pressure, so the figure only grows: a review put + * 16 GB of active cache in a container with no bee running and the guard + * answered "19.6 GB in use" every round, for good, because a memory refusal + * frees nothing. What kills a container is memory that reclaim cannot free, so + * that is what is counted: current minus the file cache, both lists, minus + * reclaimable slab. `/proc/meminfo`'s MemAvailable already means the same. + * Mapped and dirty file pages it under-counts are what the headroom is for. + * + * The limit is the cgroup's. When the cgroup says "max" the container is not + * the thing that is limited and the host's MemTotal is the real ceiling; when + * even that is not what the platform enforces, TRIOS_QUEEN_MEMORY_LIMIT_MB + * states it, and replaces the LIMIT only - use is always measured. + */ +export function readContainerMemory( + readers: { + readFile?: (path: string) => string | null + platform?: string + env?: NodeJS.ProcessEnv + } = {}, +): MemoryReading { + const platform = readers.platform ?? process.platform + if (platform !== 'linux') return { kind: 'unsupported', platform } + const read = readers.readFile ?? readFileOrNull + const stated = resourceSettings(readers.env ?? process.env).memoryLimitBytes + const meminfo = read('/proc/meminfo') + const hostTotal = meminfoBytes(meminfo, 'MemTotal') + const tried: string[] = [] + + const hostUsed = + hostTotal !== null && meminfoBytes(meminfo, 'MemAvailable') !== null + ? hostTotal - (meminfoBytes(meminfo, 'MemAvailable') ?? 0) + : null + + const finish = ( + usedRaw: number, + cgroupLimit: number | null, + source: string, + ): MemoryReading | null => { + // THE KERNEL'S LIMIT WINS. A stated limit can only LOWER the line: the + // kernel kills at its own number whatever a variable says, so a stated + // 24576 MB over a cgroup limit of 24000000000 bytes would have moved the + // guard's line to within one bee of the kill line, and said nothing. + const ceiling = cgroupLimit ?? hostTotal + const limit = + stated && ceiling ? Math.min(stated, ceiling) : (stated ?? ceiling) + if (!limit || limit <= 0) return null + const statedWins = Boolean(stated) && limit === stated + // The cgroup says "max": the container is not what is limited, the machine + // is - and the machine is also used by everything else on it. Judging this + // cgroup's own use against the host's total admitted thirteen more bees + // into a host with half a GB available. + const onHost = cgroupLimit === null && !statedWins && hostUsed !== null + const used = onHost ? Math.max(usedRaw, hostUsed ?? 0) : usedRaw + return { + kind: 'measured', + usedBytes: Math.max(0, used), + limitBytes: limit, + source: onHost && used !== usedRaw ? '/proc/meminfo' : source, + limitSource: statedWins + ? 'TRIOS_QUEEN_MEMORY_LIMIT_MB' + : cgroupLimit + ? 'cgroup' + : 'host MemTotal', + ...(stated && !statedWins + ? { + note: + `TRIOS_QUEEN_MEMORY_LIMIT_MB states ${gb(stated)} GB, above the ` + + `${gb(limit)} GB the ${cgroupLimit ? 'kernel enforces' : 'machine has'}; ` + + 'using the smaller', + } + : {}), + } + } + + tried.push('cgroup v2') + const v2Current = read('/sys/fs/cgroup/memory.current') + if (v2Current !== null && /^\d+\s*$/.test(v2Current)) { + const max = (read('/sys/fs/cgroup/memory.max') ?? '').trim() + const stat = read('/sys/fs/cgroup/memory.stat') + const givenBack = + statValue(stat, 'inactive_file') + + statValue(stat, 'active_file') + + statValue(stat, 'slab_reclaimable') + const reading = finish( + Number(v2Current) - givenBack, + /^\d+$/.test(max) ? Number(max) : null, + 'cgroup v2', + ) + if (reading) return reading + } + + tried.push('cgroup v1') + const v1Usage = read('/sys/fs/cgroup/memory/memory.usage_in_bytes') + if (v1Usage !== null && /^\d+\s*$/.test(v1Usage)) { + const limitRaw = Number( + (read('/sys/fs/cgroup/memory/memory.limit_in_bytes') ?? '').trim(), + ) + const stat = read('/sys/fs/cgroup/memory/memory.stat') + const givenBack = + statValue(stat, 'total_inactive_file') + + statValue(stat, 'total_active_file') + const reading = finish( + Number(v1Usage) - givenBack, + Number.isFinite(limitRaw) && limitRaw > 0 && limitRaw < V1_UNLIMITED + ? limitRaw + : null, + 'cgroup v1', + ) + if (reading) return reading + } + + // MEASURED from inside a Railway container, 2026-09-17: memory.max reads + // 24000000000 and /proc/meminfo reads MemTotal 338 GB - the HOST. So this last + // resort describes the whole machine, which is the right thing to judge on a + // machine with no container around the server and the wrong thing to hold + // against a stated CONTAINER limit: the host's use is always far above it and + // every start would be refused for memory this container never touched. + tried.push('/proc/meminfo') + const available = meminfoBytes(meminfo, 'MemAvailable') + if (hostTotal !== null && available !== null) { + if (stated) { + return { + kind: 'unknown', + tried, + why: + 'TRIOS_QUEEN_MEMORY_LIMIT_MB states a container limit, but only ' + + '/proc/meminfo is readable and it measures the whole host', + } + } + return { + kind: 'measured', + usedBytes: Math.max(0, hostTotal - available), + limitBytes: hostTotal, + source: '/proc/meminfo', + limitSource: 'host MemTotal', + } + } + return { kind: 'unknown', tried } +} + +/** One line saying what the guard reads, for the log - not for a refusal. */ +export function describeReading(memory: MemoryReading): string { + if (memory.kind === 'unsupported') { + return `memory is not measured on ${memory.platform}; only the volume is` + } + if (memory.kind === 'unknown') { + return `memory cannot be measured (tried ${memory.tried.join(', ')})` + } + return ( + `memory from ${memory.source}, limit ${gb(memory.limitBytes)} GB ` + + `from ${memory.limitSource}` + ) +} + +/** + * Free bytes on the volume, from `df -Pk` - the same subprocess, for the same + * reason, as `volumeUsedPercent`: statfs arithmetic is exact on the Linux + * container and wrong on an APFS shared container, and a guard that refuses + * every dispatch in a unit test is a guard somebody switches off. + */ +export function volumeSpace( + dir: string, + run: (dir: string) => string = (target) => + execFileSync('df', ['-Pk', target], { + encoding: 'utf8', + timeout: 10_000, + stdio: ['ignore', 'pipe', 'ignore'], + }), +): VolumeSpace | null { + try { + const line = run(dir).trim().split('\n').pop() ?? '' + // Filesystem 1024-blocks Used Available Capacity Mounted-on + const match = /^\S+\s+(\d+)\s+\d+\s+(\d+)\s+\d+%/.exec(line) + if (!match) return null + const totalBytes = Number(match[1]) * 1024 + const freeBytes = Number(match[2]) * 1024 + return totalBytes > 0 ? { totalBytes, freeBytes } : null + } catch { + return null + } +} + +/** + * Bees that have started and not yet grown. + * + * The tick starts bees back to back, and a bee takes minutes to allocate what + * it will hold. Thirty-four starts in one round would each see the same nearly + * empty container and each be admitted; the reading is true and the conclusion + * is wrong. So every start inside the warm-up window reserves a bee's memory + * that the reading cannot show yet. + * + * Process-local on purpose: after a restart every bee has been reaped, so an + * empty list is the correct list. It reserves memory; it caps nothing by count. + */ +const youngBees = new Map() +/** No warm-up this module accepts is longer; past it an entry is only garbage. */ +const LONGEST_WARMUP_MS = 3600 * 1000 + +/** + * Keyed by the turn, because a reservation must end with the bee. `turn.ok` + * only means /chat answered 200, and a provider refusal arrives exactly that + * way: 200, then the failure in the stream. Kept as bare timestamps, ten such + * dead starts in ten minutes reserved ten GB for bees that no longer existed + * and the guard refused a nearly empty container, saying they were "still + * growing". `closeDispatch`, where every ending lands, calls `noteBeeEnded`. + */ +export function noteBeeStarted(turn: string, nowMs: number): void { + youngBees.set(turn, nowMs) +} + +export function noteBeeEnded(turn: string): void { + youngBees.delete(turn) +} + +export function youngBeeCount(nowMs: number, warmupSeconds: number): number { + const horizon = nowMs - warmupSeconds * 1000 + let young = 0 + for (const [turn, startedAt] of youngBees) { + // Dropped only past the LONGEST window, never past the caller's: the window + // is read on every call and may grow, and a count must not destroy what a + // later count would have needed. + if (nowMs - startedAt > LONGEST_WARMUP_MS) youngBees.delete(turn) + else if (startedAt > horizon) young += 1 + } + return warmupSeconds > 0 ? young : 0 +} + +export function resetYoungBees(): void { + youngBees.clear() +} + +/** + * One more bee: does it fit? + * + * Pure. Memory is judged first because it is the failure that takes everything + * with it, and ONE cause is reported - a sentence naming two is a sentence an + * operator acts on half of. Unknown is a refusal: a guard that reads an + * unmeasurable container as empty disables itself exactly when the container + * is unwell. Not-Linux passes, because there is no container to protect. + * + * The FIRST SENTENCE carries every number, names no variable and stays under + * 200 characters, because it is the part the round report quotes as the reason + * nothing started (`containerRefusal` in queen-report-lines.ts). What to do + * about it comes after. + */ +export function judgeBeeRoom(input: { + memory: MemoryReading + volume: VolumeSpace | null + youngBees: number + settings: ResourceSettings + volumeDir: string + reaped?: { removed: number; keptDirty: number; keptRunning: number } + /** + * The workspace directory does not exist: there is no checkout to fill, and + * `df` on a missing path fails like a sick filesystem does. The entrypoint + * starts the server even when the clone failed, and "no room in the + * container: disk" with nineteen GB free sent the operator to the wrong + * place while hiding the git error that names the real one. So disk is not + * judged, and `prepareWorktree` says what is actually wrong. + */ + checkoutMissing?: boolean +}): BeeRoom { + const { memory, volume, youngBees, settings, volumeDir, reaped } = input + + const refuse = ( + resource: 'memory' | 'disk', + summary: string, + advice: string, + ): BeeRoom => ({ + ok: false, + resource, + summary, + detail: `${summary}. ${advice}`, + }) + + if (memory.kind === 'unknown') { + return refuse( + 'memory', + `container memory cannot be measured (tried ${memory.tried.join(', ')}) ` + + 'and unknown is not room', + `${memory.why ? `${memory.why}. ` : ''}Not starting a bee blind; ` + + 'TRIOS_QUEEN_RESOURCE_GUARD=off dispatches without the guard.', + ) + } + if (memory.kind === 'measured') { + const keepFree = Math.min( + (memory.limitBytes * settings.memoryHeadroomPercent) / 100, + MEMORY_HEADROOM_CAP, + ) + const asked = (youngBees + 1) * settings.beeMemoryBytes + if (memory.usedBytes + asked > memory.limitBytes - keepFree) { + // Short on purpose: the first sentence is what a report line shows. + const where = + memory.limitSource === 'cgroup' + ? memory.source + : memory.limitSource === 'host MemTotal' + ? `${memory.source}, host total` + : `${memory.source}, stated limit` + // The window as it was set: "0 min" for twenty seconds told an operator + // the count was taken over nothing. + const window = + settings.warmupSeconds % 60 === 0 + ? `${settings.warmupSeconds / 60} min` + : `${settings.warmupSeconds} s` + const growing = + youngBees > 0 + ? ` and ${youngBees} bee(s) started in the last ${window} are still growing` + : '' + return refuse( + 'memory', + `container memory is ${gb(memory.usedBytes)} GB of ` + + `${gb(memory.limitBytes)} GB in use (${where})${growing}; another bee ` + + `needs about ${gb(settings.beeMemoryBytes)} GB and ${gb(keepFree)} GB ` + + 'stays free', + 'Not starting a bee that would get every running bee killed. Asked ' + + 'again every round, and a finishing bee is what gives memory back; ' + + 'TRIOS_QUEEN_BEE_MEMORY_MB and TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT ' + + '(MiB and percent) move the line.', + ) + } + } + + if (input.checkoutMissing) return { ok: true } + if (volume === null) { + return refuse( + 'disk', + `free space on ${volumeDir} cannot be measured (df gave nothing) and ` + + 'unknown is not room', + 'Not starting a bee blind; TRIOS_QUEEN_RESOURCE_GUARD=off dispatches ' + + 'without the guard.', + ) + } + const keepFree = diskKeptFree(volume, settings) + if (volume.freeBytes < settings.beeDiskBytes + keepFree) { + const reaping = reaped + ? `after a reap (${reaped.removed} removed; kept ${reaped.keptDirty} with ` + + `uncommitted work and ${reaped.keptRunning} of running bees)` + : '(nothing reaped yet)' + return refuse( + 'disk', + `volume ${volumeDir} has ${gb(volume.freeBytes)} GB free of ` + + `${gb(volume.totalBytes)} GB ${reaping}; another bee needs about ` + + `${gb(settings.beeDiskBytes)} GB and ${gb(keepFree)} GB stays free`, + 'Not starting a bee that would fill the disk under the running ones. ' + + 'TRIOS_QUEEN_BEE_DISK_MB and TRIOS_QUEEN_DISK_HEADROOM_PERCENT move the line.', + ) + } + return { ok: true } +} + +function diskKeptFree(volume: VolumeSpace, settings: ResourceSettings): number { + return Math.min( + (volume.totalBytes * settings.diskHeadroomPercent) / 100, + DISK_HEADROOM_CAP, + ) +} + +/** + * The used-percent at which the disk rule is satisfied, for the reaper to stop + * at. Without it the one reap the guard asks for ran to QUEEN_VOLUME_LOW - 55% + * - and removed every clean tree it was allowed to, when the guard needed seven + * GB. Rounded down, so stopping there is never stopping short. + */ +export function diskLineUsedPercent( + volume: VolumeSpace, + settings: ResourceSettings, +): number { + const need = settings.beeDiskBytes + diskKeptFree(volume, settings) + return Math.max(0, Math.floor(100 * (1 - need / volume.totalBytes))) +} diff --git a/trios/agent-server/apps/server/src/api/services/queen-tick.ts b/trios/agent-server/apps/server/src/api/services/queen-tick.ts index dc6cb55f85..65ad80f59e 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-tick.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-tick.ts @@ -60,6 +60,7 @@ import { releaseQueenLease, } from './queen-lease' import { + containerRefusal, type DispatchReportOutcome, dispatchesThatStarted, nothingStartedLine, @@ -1488,6 +1489,7 @@ export async function runRound( detail: string conversationId?: string keyIndex?: number + room?: { resource: 'memory' | 'disk'; summary: string } }> = [] let board = [...registry.rows[0].tasks, ...containerTasks] let current: QueendChoice | null = choice @@ -2589,18 +2591,29 @@ async function report( '.', ) } + const noRoom = containerRefusal(started) if (dispatchesThatStarted(started).length === 0) { // The refusal, verbatim. A round that started nothing is the case where a // summary in my own words would be the least trustworthy thing on the page. // This fires for a round that dispatched and was refused as it does for a // round that never dispatched - the sentence is the same because the fact - // (no bee started) is the same. The refused dispatches above carry the why. - lines.push(nothingStartedLine(choice.refusal, candidates)) + // (no bee started) is the same. When `queend` allowed the round and the + // CONTAINER refused it, the why is the guard's sentence, and it goes here + // as well as in the line above: "No reason given" next to a reason is a + // report contradicting itself. + lines.push( + nothingStartedLine(choice.refusal ?? noRoom?.sentence, candidates), + ) const skipped = (choice.skipped ?? []).slice(0, 6) if (skipped.length > 0) lines.push('', ...skipped.map((s) => ` ${s}`)) } - const headline = reportHeadline(escalated.length, started, choice.refusal) + // The label, never the sentence: this string is served to a browser. + const headline = reportHeadline( + escalated.length, + started, + choice.refusal ?? noRoom?.headline, + ) await pool .query( diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index a6e0a466c6..94e9a1d841 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -1,5 +1,18 @@ -import { afterEach, beforeAll, describe, expect, it } from 'bun:test' -import { mkdirSync, mkdtempSync, realpathSync, writeFileSync } from 'node:fs' +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'bun:test' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { Pool } from 'pg' @@ -18,12 +31,14 @@ import { missingProviderRefusal, POOL_KEY_STRIDE, prepareWorktree, + reapWorktrees, recordDispatch, resolveWorkerProvider, setDurableCloseListener, workerCapacityBreakdown, workspaceRoot, } from '../../src/api/services/queen-dispatch' +import { resetYoungBees } from '../../src/api/services/queen-resources' import { logger } from '../../src/lib/logger' const GENERIC_WORKER_KEYS = [ @@ -67,6 +82,16 @@ const KEYS = [ 'TRIOS_QUEEN_WORKER_CONTEXT', 'TRIOS_QUEEN_WORKER_LANES_PER_KEY', 'TRIOS_QUEEN_MAX_WORKERS', + // Every variable the container guard reads. One of them left in a developer's + // shell moves the line the fixtures below are written against, and a fixture + // that stops being "full" walks past the gate into a real `prepareWorktree`. + 'TRIOS_QUEEN_RESOURCE_GUARD', + 'TRIOS_QUEEN_BEE_MEMORY_MB', + 'TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT', + 'TRIOS_QUEEN_BEE_WARMUP_SECONDS', + 'TRIOS_QUEEN_BEE_DISK_MB', + 'TRIOS_QUEEN_DISK_HEADROOM_PERCENT', + 'TRIOS_QUEEN_MEMORY_LIMIT_MB', // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. 'TRIOS_QUEEN_WORKER_API_KEY_17', 'TRIOS_QUEEN_WORKER_API_KEY_1024', @@ -744,6 +769,464 @@ describe('queen dispatch precheck', () => { }) }) +/** + * A key is free and a provider answers - and the container may still have no + * room. Measured on 2026-09-17: 24 GB, about 0.85 GB a bee, thirty-four lanes + * connected. Nothing in dispatch asked the container before this. + */ +describe('the container is asked before a worktree is cut', () => { + // Decimal, like every number the guard prints. + const GB = 1_000_000_000 + const full = () => + ({ + kind: 'measured', + usedBytes: 21.2 * GB, + limitBytes: 24 * GB, + source: 'cgroup v2', + limitSource: 'cgroup', + }) as const + const calm = () => ({ ...full(), usedBytes: 3 * GB }) + const roomy = () => ({ totalBytes: 50 * GB, freeBytes: 19 * GB }) + const oneFreeKey = () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'https://api.z.ai/api/paas/v4' + process.env.TRIOS_QUEEN_WORKER_API_KEY = 'a' + } + // An EMPTY directory stands where the workspace would be, for every case. A + // fixture that stops reading as "full" - a variable in somebody's shell moved + // the line - walks past the gate into the real `prepareWorktree`, and on the + // container the default workspace is the LIVE checkout: a unit test would + // fetch there and cut a worktree in it. + let previousWorkspace: string | undefined + beforeEach(() => { + previousWorkspace = process.env.WORKSPACE_DIR + process.env.WORKSPACE_DIR = realpathSync( + mkdtempSync(join(tmpdir(), 'queen-guard-')), + ) + }) + afterEach(() => { + if (previousWorkspace === undefined) delete process.env.WORKSPACE_DIR + else process.env.WORKSPACE_DIR = previousWorkspace + // Module state, shared by every test file in the bun process. + resetYoungBees() + }) + + const recordingPool = (running: number[] = []) => { + const queries: Array<{ text: string; values?: unknown[] }> = [] + const pool = { + query: async (text: string, values?: unknown[]) => { + queries.push({ text, values }) + return /finished_at IS NULL/.test(text) + ? { + rowCount: running.length, + rows: running.map((issue) => ({ issue })), + } + : { rowCount: 1, rows: [] } + }, + } as unknown as Pool + return { pool, queries } + } + + it('refuses on memory with the numbers, cuts nothing, and books nothing against the issue', async () => { + oneFreeKey() + const { pool, queries } = recordingPool() + let reaped = 0 + + const outcome = await dispatchBee( + pool, + 1500, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: full, + volume: roomy, + reap: async () => { + reaped += 1 + return { + before: 0, + after: 0, + removed: [], + keptDirty: [], + keptRunning: [], + refused: [], + } + }, + }, + ) + + expect(outcome.started).toBe(false) + expect(outcome.detail).toMatch( + /^container memory is 21\.2 GB of 24\.0 GB in use/, + ) + // Tagged with the guard's own sentence, because the report may quote this + // refusal and no other. + expect(outcome.room?.resource).toBe('memory') + expect(outcome.detail.startsWith(`${outcome.room?.summary}. `)).toBe(true) + // Memory cannot be freed by reaping, so the reaper is not even asked. + expect(reaped).toBe(0) + // It says something about the CONTAINER, and once the swarm is memory-bound + // it is how nearly every round ends. A row per round would archive history + // and wipe the issue's last attempt for a reason that is not the issue's. + expect(queries.filter((q) => /queen_dispatch/.test(q.text))).toEqual([]) + }) + + it('gives the disk one reap that protects running bees and its own tree, then names what was kept', async () => { + oneFreeKey() + const { pool } = recordingPool([7, 12]) + const calls: Array<{ + protect?: ReadonlySet + high?: number + low?: number + }> = [] + + const outcome = await dispatchBee( + pool, + 1501, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => ({ totalBytes: 50 * GB, freeBytes: 5.1 * GB }), + reap: async (opts = {}) => { + calls.push(opts) + return { + before: 90, + after: 90, + removed: ['/w/.worktrees/queen-3'], + keptDirty: [], + keptRunning: ['/w/.worktrees/queen-7', '/w/.worktrees/queen-12'], + refused: [], + } + }, + }, + ) + + expect(calls).toHaveLength(1) + // queen-1501 is the issue being dispatched. It is not running, so without + // its own name here a re-dispatch could reap the tree it was about to reuse + // and then cut it afresh with -B, dropping the last attempt's commits. + expect([...(calls[0].protect ?? [])].sort()).toEqual([ + 'queen-12', + 'queen-1501', + 'queen-7', + ]) + // It starts whatever the percentage (high 0) and stops where the rule is + // met - 2048 MiB + 5 GB of 50 GB is 85% used - not at the collector's 55%. + expect(calls[0].high).toBe(0) + expect(calls[0].low).toBe(85) + expect(outcome.started).toBe(false) + expect(outcome.room?.resource).toBe('disk') + expect(outcome.detail).toContain( + 'has 5.1 GB free of 50.0 GB after a reap (1 removed; kept 0 with uncommitted work and 2 of running bees)', + ) + }) + + it('reaps nothing when it cannot find out who is running', async () => { + // An unreadable registry used to mean "protect nobody": with twenty bees + // running, that reap takes the clean trees out from under most of them. + oneFreeKey() + const pool = { + query: async (text: string) => { + if (/finished_at IS NULL/.test(text)) { + throw new Error('Connection terminated unexpectedly') + } + return { rowCount: 1, rows: [] } + }, + } as unknown as Pool + let reaps = 0 + const outcome = await dispatchBee( + pool, + 1505, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => ({ totalBytes: 50 * GB, freeBytes: 5.1 * GB }), + reap: async () => { + reaps += 1 + throw new Error('the reaper must not be called') + }, + }, + ) + expect(reaps).toBe(0) + expect(outcome.started).toBe(false) + expect(outcome.room?.resource).toBe('disk') + expect(outcome.detail).toContain('(nothing reaped yet)') + }) + + it('starts the bee when the one reap made the room', async () => { + // The half of the disk path that ADMITS. With it broken - the second + // judgement reusing the first reading, or refusing whatever it reads - a + // volume past the line can never recover: the gate sits in front of the only + // reaper there was, so every round would refuse on disk for good. + oneFreeKey() + const { pool } = recordingPool() + let reaps = 0 + const outcome = await dispatchBee( + pool, + 1504, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => + reaps === 0 ? { totalBytes: 50 * GB, freeBytes: 5.1 * GB } : roomy(), + reap: async () => { + reaps += 1 + return { + before: 90, + after: 60, + removed: ['/w/.worktrees/queen-3'], + keptDirty: [], + keptRunning: [], + refused: [], + } + }, + }, + ) + expect(reaps).toBe(1) + // Past the gate: what stops it next is the empty workspace, not the guard. + expect(outcome.room).toBeUndefined() + expect(outcome.detail).not.toContain('GB free of') + }) + + /** A repository to cut worktrees from, standing where the workspace is. */ + const hive = (): { root: string; git: (args: string[]) => string } => { + const root = join(process.env.WORKSPACE_DIR ?? '', 'BrowserOS') + const git = (args: string[], cwd = root) => { + const done = Bun.spawnSync(['git', ...args], { + cwd, + env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' }, + }) + if (done.exitCode !== 0) { + throw new Error( + `git ${args.join(' ')} failed: ${done.stderr.toString()}`, + ) + } + return done.stdout.toString().trim() + } + mkdirSync(root) + git(['init', '-q', '-b', 'dev']) + git(['config', 'user.email', 'bee@example.com']) + git(['config', 'user.name', 'Bee']) + writeFileSync(join(root, 'README.md'), 'hive\n') + git(['add', '.']) + git(['-c', 'commit.gpgsign=false', 'commit', '-qm', 'first']) + git(['remote', 'add', 'origin', root]) + git(['fetch', '-q', 'origin']) + return { root, git } + } + /** Everything a real start reads from the environment, put back afterwards. */ + const realStarts = () => { + const names = [ + 'TRIOS_API_TOKEN', + 'TRIOS_REPO_URL', + 'TRIOS_REPO_REF', + 'QUEEN_VOLUME_KEEP', + ] + const previous = names.map((name) => process.env[name]) + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + delete process.env.TRIOS_REPO_URL + delete process.env.TRIOS_REPO_REF + const realFetch = globalThis.fetch + // /chat answers 200 and the stream STAYS OPEN, as a working bee's does: a + // stream that is already over ends the bee, and an ended bee reserves + // nothing. + const streams: Array> = [] + globalThis.fetch = (async () => + new Response( + new ReadableStream({ + start(controller) { + streams.push(controller) + }, + }), + )) as unknown as typeof fetch + const finish = async (index: number) => { + streams[index].enqueue( + new TextEncoder().encode('data: {"type":"finish"}\n\n'), + ) + streams[index].close() + delete streams[index] + // The drain closes the dispatch on its own turn of the loop. + await new Promise((resolve) => setTimeout(resolve, 50)) + } + return { + finish, + restore: async () => { + globalThis.fetch = realFetch + for (const index of Object.keys(streams)) await finish(Number(index)) + names.forEach((name, i) => { + if (previous[i] === undefined) delete process.env[name] + else process.env[name] = previous[i] + }) + }, + } + } + + it('reserves for the bee it just started, and gives it back the moment that bee ends', async () => { + // The wiring, not the arithmetic: a start that is not noted, or a judgement + // that does not count the young, admits thirty-four bees back to back on + // one true reading of a nearly empty container - the incident this exists + // to prevent. So this drives REAL starts: a repository to cut the worktree + // from, and `/chat` answering 200. + oneFreeKey() + hive() + const turns = realStarts() + const { pool } = recordingPool() + // 19.7 of 24 GB: one more bee fits under the line (21.6), two do not. + const nearlyFull = () => ({ ...full(), usedBytes: 19.7 * GB }) + const start = 1_789_000_000_000 + const go = (issue: number, ms: number) => + dispatchBee(pool, issue, 'brief', [], [], undefined, [], 'none', { + memory: nearlyFull, + volume: roomy, + now: () => ms, + // Not this machine's disk: at 95% full the cut itself is refused. + volumeUsed: () => 40, + }) + try { + const first = await go(1510, start) + expect(first.detail).toContain('zai/') + expect(first.started).toBe(true) + + const second = await go(1511, start + 60_000) + expect(second.started).toBe(false) + expect(second.room?.resource).toBe('memory') + expect(second.room?.summary).toContain( + '1 bee(s) started in the last 10 min are still growing', + ) + + // The first bee ends - a provider refusal ends one in seconds - and its + // reservation ends with it, two minutes in and not ten. + await turns.finish(0) + const third = await go(1512, start + 120_000) + expect(third.started).toBe(true) + + // The third is still running and still young: no room for another... + expect((await go(1513, start + 180_000)).started).toBe(false) + // ...until it has had its ten minutes, and its memory is in the reading. + expect((await go(1513, start + 13 * 60_000)).started).toBe(true) + } finally { + await turns.restore() + } + }) + + it('never reaps the tree of the issue it is about to dispatch again', async () => { + // A redeploy killed the swarm; #1520 was reaped at boot and its tree holds + // one committed, unpushed attempt. The container carries no push + // credential, so that commit is the only copy. With the volume short the + // gate reaps - and #1520 is by definition not running. Reaped, its tree + // would be cut afresh with -B and the commit would drop off the branch. + oneFreeKey() + const { root, git } = hive() + const turns = realStarts() + process.env.QUEEN_VOLUME_KEEP = '0' + for (const branch of ['queen-1520', 'queen-3']) { + git([ + 'worktree', + 'add', + '-q', + '-b', + branch, + join(root, '.worktrees', branch), + 'dev', + ]) + } + const tree = join(root, '.worktrees', 'queen-1520') + writeFileSync(join(tree, 'attempt.md'), 'the only copy\n') + Bun.spawnSync(['git', 'add', '.'], { cwd: tree }) + Bun.spawnSync( + [ + 'git', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'the first attempt', + ], + { cwd: tree, env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' } }, + ) + const attempt = git(['rev-parse', 'queen-1520']) + const { pool } = recordingPool() + let reaps = 0 + try { + const outcome = await dispatchBee( + pool, + 1520, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => + reaps === 0 + ? { totalBytes: 50 * GB, freeBytes: 5.1 * GB } + : roomy(), + // The real reaper against the real repository, on a fixed reading. + reap: async (opts = {}) => { + reaps += 1 + return reapWorktrees({ ...opts, volumeUsed: () => 90 }) + }, + volumeUsed: () => 40, + }, + ) + expect(reaps).toBe(1) + // The reap did run, and did remove what nobody needs... + expect(existsSync(join(root, '.worktrees', 'queen-3'))).toBe(false) + // ...and the issue's own tree was reused with its commit where it was. + expect(outcome.detail).toMatch(/^reused an existing worktree/) + expect(git(['rev-parse', 'queen-1520'])).toBe(attempt) + expect(outcome.started).toBe(true) + } finally { + await turns.restore() + } + }) + + it('can be switched off, and then a full container refuses nothing here', async () => { + oneFreeKey() + process.env.TRIOS_QUEEN_RESOURCE_GUARD = 'off' + const { pool } = recordingPool() + const outcome = await dispatchBee( + pool, + 1503, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: full, + volume: () => null, + }, + ) + // Whatever stops it next (there is no repository in this directory), it is + // not the guard. + expect(outcome.room).toBeUndefined() + expect(outcome.detail).not.toContain('container memory is') + expect(outcome.detail).not.toContain('unknown is not room') + }) +}) + /** * #1308. `workers.capacity` answers a number; this breakdown answers what the * number is MADE of. An operator seeing capacity 4 cannot act on it without diff --git a/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts b/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts index f23b1c7766..da31b8b602 100644 --- a/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts @@ -6,18 +6,25 @@ * The #1379 regression test: a round whose dispatches were all refused must * report that nothing started, not report a bee working. * - * Imports `bun:test` and the module under test and nothing else. The module - * has no imports by design - a bee's worktree has no `node_modules`, so this - * test has to run with neither a database nor the `queend` binary to hand. + * Imports `bun:test`, the module under test and one neighbour that itself + * imports only Node builtins. The module has no imports by design - a bee's + * worktree has no `node_modules`, so this test has to run with neither a + * database nor the `queend` binary to hand. */ import { describe, expect, it } from 'bun:test' import { + containerRefusal, dispatchesThatStarted, nothingStartedLine, refusedLines, reportHeadline, startedLine, } from '../../src/api/services/queen-report-lines' +// Builtins only, like the module above: it loads in a bare worktree too. +import { + judgeBeeRoom, + resourceSettings, +} from '../../src/api/services/queen-resources' /** The refusal `dispatchBee` returns when every provider key is busy - the * routine case from the issue, not an exotic failure. */ @@ -131,3 +138,127 @@ describe('queen report dispatch lines', () => { expect(short).toBe('Refused #8: no.') }) }) + +describe('a round queend allowed and the container refused', () => { + const GB = 1_000_000_000 + const settings = resourceSettings({}) + /** What the guard REALLY returns, at the sizes that make it longest: a + * hand-shortened sample is how a cut in the middle of a number went unseen. */ + const refusal = (room: ReturnType, issue = 1500) => { + if (room.ok) throw new Error('the fixture was meant to have no room') + return { + started: false, + issue, + detail: room.detail, + room: { resource: room.resource, summary: room.summary }, + } + } + const memoryFull = () => + judgeBeeRoom({ + memory: { + kind: 'measured', + usedBytes: 1234.5 * GB, + limitBytes: 4096 * GB, + source: '/proc/meminfo', + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 1000, + settings: { ...settings, beeMemoryBytes: 64 * GB, warmupSeconds: 3599 }, + volumeDir: '/workspace/BrowserOS', + }) + const diskShort = ( + volumeDir = '/workspace/BrowserOS', + reaped = { removed: 1234, keptDirty: 567, keptRunning: 1024 }, + ) => + judgeBeeRoom({ + memory: { kind: 'unsupported', platform: 'darwin' }, + volume: { totalBytes: 50 * GB, freeBytes: 5.1 * GB }, + youngBees: 0, + settings, + volumeDir, + reaped, + }) + + it('quotes the whole summary in the line that read "No reason given"', () => { + for (const room of [memoryFull(), diskShort()]) { + const said = containerRefusal([refusal(room)]) + // Whole: it ends where the guard's sentence ends, not mid-word. + expect(said?.sentence.endsWith('GB stays free')).toBe(true) + expect(said?.sentence.length).toBeLessThanOrEqual(200) + const line = nothingStartedLine(said?.sentence, 4) + expect(line).not.toContain('No reason given') + expect(line).toContain('GB stays free. 4 issue(s) were on the table.') + } + }) + + it('is not fooled by a path that holds a period and a space', () => { + // The sentence used to be cut out of the prose at the first ". ", and the + // prose names the workspace: the report read "volume /Volumes/Ext." + const said = containerRefusal([ + refusal( + diskShort('/Volumes/Ext. Drive/ws/BrowserOS', { + removed: 2, + keptDirty: 1, + keptRunning: 14, + }), + ), + ]) + expect(said?.sentence).toContain( + '/Volumes/Ext. Drive/ws/BrowserOS has 5.1 GB', + ) + expect(said?.sentence.endsWith('GB stays free')).toBe(true) + }) + + it('gives the headline a label and never the sentence', () => { + const outcomes = [refusal(diskShort())] + const headline = reportHeadline( + 0, + outcomes, + containerRefusal(outcomes)?.headline, + ) + expect(headline).toBe('no room in the container: disk') + // /queen/needs-you serves this string to a browser: no path, no numbers. + expect(headline).not.toContain('/workspace') + }) + + it('reads no other refusal, because those details carry git and provider output', () => { + expect( + containerRefusal([ + { + started: false, + issue: 7, + detail: + "git worktree add failed: fatal: '/workspace/BrowserOS/.worktrees/queen-7' already exists", + }, + { started: false, issue: 8, detail: KEY_EXHAUSTED_DETAIL }, + ]), + ).toBeNull() + expect( + containerRefusal([{ started: true, issue: 1, detail: 'fresh worktree' }]), + ).toBeNull() + expect(containerRefusal([])).toBeNull() + }) + + it('closes a summary that is too long anyway with the ellipsis this file uses', () => { + const said = containerRefusal([ + { + started: false, + issue: 9, + detail: 'irrelevant', + room: { + resource: 'disk', + summary: `volume /${'deep/'.repeat(60)} has 1.0 GB free`, + }, + }, + ]) + expect(said?.sentence.length).toBeLessThanOrEqual(200) + expect(nothingStartedLine(said?.sentence, 1)).toContain('... 1 issue(s)') + // And a refusal that somehow carries no summary still says something true. + expect( + containerRefusal([ + { started: false, issue: 1, room: { resource: 'memory' } }, + ])?.sentence, + ).toBe('no room in the container: memory') + }) +}) diff --git a/trios/agent-server/apps/server/tests/api/queen-resources.test.ts b/trios/agent-server/apps/server/tests/api/queen-resources.test.ts new file mode 100644 index 0000000000..429a9c7af6 --- /dev/null +++ b/trios/agent-server/apps/server/tests/api/queen-resources.test.ts @@ -0,0 +1,595 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * WHY THIS EXISTS. + * + * 2026-09-17: the container is limited to 24 GB and held 14.7 GB with sixteen + * bees running - about 0.85 GB a bee. The same day the worker ceiling stopped + * being a constant and the credential list reached thirty-four lanes. A full + * swarm would have asked for about thirty GB, and the kernel answers that by + * killing the container with every running bee in it. Nothing in dispatch + * looked at memory at all. + * + * These pin the judge with the container in states a test machine never is in: + * every reading is injected, so none of this needs Linux, a cgroup or a full + * disk to be true. + */ +import { afterEach, describe, expect, it } from 'bun:test' +import { + describeReading, + diskLineUsedPercent, + judgeBeeRoom, + type MemoryReading, + noteBeeEnded, + noteBeeStarted, + readContainerMemory, + resetYoungBees, + resourceSettings, + volumeSpace, + youngBeeCount, +} from '../../src/api/services/queen-resources' + +// DECIMAL, like every number the module prints and the platform sells: the +// Railway container is 24000000000 bytes and everyone calls it 24 GB. +const GB = 1_000_000_000 +const files = (map: Record) => (path: string) => + path in map ? map[path] : null +const measured = (usedGb: number, limitGb = 24): MemoryReading => ({ + kind: 'measured', + usedBytes: usedGb * GB, + limitBytes: limitGb * GB, + source: 'cgroup v2', + limitSource: 'cgroup', +}) +const roomy = { totalBytes: 50 * GB, freeBytes: 19 * GB } +const defaults = resourceSettings({}) + +afterEach(() => resetYoungBees()) + +describe('reading the container', () => { + it('counts what the kernel cannot give back, not the cache it will', () => { + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '16000000000\n', + '/sys/fs/cgroup/memory.max': '25769803776\n', + '/sys/fs/cgroup/memory.stat': + 'anon 9\ninactive_file 2000000000\nactive_file 5\n', + }), + }) + expect(reading).toEqual({ + kind: 'measured', + usedBytes: 16_000_000_000 - 2_000_000_000 - 5, + limitBytes: 25_769_803_776, + source: 'cgroup v2', + limitSource: 'cgroup', + }) + }) + + it('does not refuse an idle container for a day of page cache', () => { + // No bee running: 2.5 GB of heap, and 16 GB of ACTIVE cache plus 2.5 GB of + // reclaimable slab left by a day of clones and installs. Counted as the + // kubelet counts (current - inactive_file) that is 21 GB "in use", past the + // line, and a memory refusal frees nothing - so every round refused until a + // redeploy. + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '23000000000', + '/sys/fs/cgroup/memory.max': '24000000000', + '/sys/fs/cgroup/memory.stat': + 'anon 2500000000\nfile 18000000000\ninactive_file 2000000000\n' + + 'active_file 16000000000\nslab_reclaimable 2500000000\nslab_unreclaimable 100\n', + }), + }) + expect(reading).toMatchObject({ usedBytes: 2_500_000_000 }) + expect( + judgeBeeRoom({ + memory: reading, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }), + ).toEqual({ ok: true }) + }) + + it('takes the host as the ceiling when the cgroup says max, and the variable when stated', () => { + const readFile = files({ + '/sys/fs/cgroup/memory.current': '3000000000', + '/sys/fs/cgroup/memory.max': 'max', + '/proc/meminfo': + 'MemTotal: 67108864 kB\nMemAvailable: 60000000 kB\n', + }) + const host = readContainerMemory({ platform: 'linux', env: {}, readFile }) + expect(host).toMatchObject({ + limitBytes: 67_108_864 * 1024, + limitSource: 'host MemTotal', + }) + // The variable replaces the LIMIT only. Use is still measured. + const stated = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile, + }) + expect(stated).toMatchObject({ + usedBytes: 3_000_000_000, + limitBytes: 24_576 * 1024 * 1024, + source: 'cgroup v2', + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }) + }) + + it('reads cgroup v1, and treats its near-2^63 limit as no limit', () => { + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory/memory.usage_in_bytes': '5000000000', + '/sys/fs/cgroup/memory/memory.limit_in_bytes': '9223372036854771712', + '/sys/fs/cgroup/memory/memory.stat': 'total_inactive_file 1000000000\n', + '/proc/meminfo': 'MemTotal: 33554432 kB\nMemAvailable: 31457280 kB\n', + }), + }) + expect(reading).toMatchObject({ + usedBytes: 4_000_000_000, + limitBytes: 33_554_432 * 1024, + source: 'cgroup v1', + limitSource: 'host MemTotal', + }) + }) + + it('judges the MACHINE when the cgroup says max, because everything else on it uses it too', () => { + // An unlimited container on a shared 16 GB machine: this cgroup holds one + // GB, the rest of the machine holds fifteen. Its own use against the + // host's total admitted thirteen more bees into half a GB. + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '1000000000', + '/sys/fs/cgroup/memory.max': 'max', + '/proc/meminfo': 'MemTotal: 16000000 kB\nMemAvailable: 500000 kB\n', + }), + }) + expect(reading).toMatchObject({ + usedBytes: 15_500_000 * 1024, + limitBytes: 16_000_000 * 1024, + source: '/proc/meminfo', + limitSource: 'host MemTotal', + }) + }) + + it('lets a stated limit lower the line and never raise it past what the kernel enforces', () => { + const railway = files({ + '/sys/fs/cgroup/memory.current': '23500000000', + '/sys/fs/cgroup/memory.max': '24000000000', + '/proc/meminfo': 'MemTotal: 338368112 kB\nMemAvailable: 109626516 kB\n', + }) + // 24576 MiB is 25.8 GB: the natural way to write "24 GB", and above the + // kernel's 24000000000. Taken at its word it admitted a bee at 23.5 GB. + const above = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile: railway, + }) + expect(above).toMatchObject({ + limitBytes: 24_000_000_000, + limitSource: 'cgroup', + }) + if (above.kind !== 'measured') throw new Error('expected a reading') + expect(above.note).toContain( + 'states 25.8 GB, above the 24.0 GB the kernel enforces', + ) + expect( + judgeBeeRoom({ + memory: above, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }).ok, + ).toBe(false) + // Below it, the variable is the line and says so. + const below = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '16384' }, + readFile: railway, + }) + expect(below).toMatchObject({ + limitBytes: 16_384 * 1024 * 1024, + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }) + if (below.kind === 'measured') expect(below.note).toBeUndefined() + }) + + it('falls back to /proc/meminfo, floors use at zero, and says unknown when nothing reads', () => { + expect( + readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/proc/meminfo': 'MemTotal: 1000 kB\nMemAvailable: 400 kB\n', + }), + }), + ).toMatchObject({ usedBytes: 600 * 1024, source: '/proc/meminfo' }) + expect( + readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '1024' }, + readFile: files({ + '/sys/fs/cgroup/memory.current': '100', + '/sys/fs/cgroup/memory.stat': 'inactive_file 500\n', + }), + }), + ).toMatchObject({ usedBytes: 0 }) + expect( + readContainerMemory({ platform: 'linux', env: {}, readFile: () => null }), + ).toEqual({ + kind: 'unknown', + tried: ['cgroup v2', 'cgroup v1', '/proc/meminfo'], + }) + }) + + it('will not hold the HOST against a stated container limit', () => { + // Measured inside a Railway container on 2026-09-17: memory.max reads + // 24000000000 while /proc/meminfo reads the machine - MemTotal 338 GB, 228 + // of them in use. Judged against a stated 24 GB that is "full" forever. + const railwayHost = + 'MemTotal: 338368112 kB\nMemAvailable: 109626516 kB\n' + const reading = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile: files({ '/proc/meminfo': railwayHost }), + }) + expect(reading.kind).toBe('unknown') + const refusal = judgeBeeRoom({ + memory: reading, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }) + expect(refusal.ok).toBe(false) + if (!refusal.ok) { + expect(refusal.detail).toContain('it measures the whole host') + expect(refusal.detail).toContain('TRIOS_QUEEN_RESOURCE_GUARD=off') + } + // With nothing stated the same files ARE the machine, and that is judged. + expect( + readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ '/proc/meminfo': railwayHost }), + }), + ).toMatchObject({ kind: 'measured', limitSource: 'host MemTotal' }) + // And what Railway really shows is read as the container it is. + const container = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '26374144', + '/sys/fs/cgroup/memory.max': '24000000000', + '/sys/fs/cgroup/memory.stat': 'inactive_file 2363392\n', + '/proc/meminfo': railwayHost, + }), + }) + expect(container).toMatchObject({ + usedBytes: 26_374_144 - 2_363_392, + limitBytes: 24_000_000_000, + limitSource: 'cgroup', + }) + expect(describeReading(container)).toBe( + 'memory from cgroup v2, limit 24.0 GB from cgroup', + ) + }) + + it('has nothing to read on a machine that is not Linux, and says which', () => { + expect(readContainerMemory({ platform: 'darwin' })).toEqual({ + kind: 'unsupported', + platform: 'darwin', + }) + }) + + it('parses df -Pk, and returns null - not zero - when it cannot', () => { + const line = + 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/x 52428800 32505856 19922944 62% /workspace\n' + expect(volumeSpace('/workspace', () => line)).toEqual({ + totalBytes: 52_428_800 * 1024, + freeBytes: 19_922_944 * 1024, + }) + expect(volumeSpace('/workspace', () => 'garbage')).toBeNull() + expect( + volumeSpace('/workspace', () => { + throw new Error('df is not there') + }), + ).toBeNull() + expect(volumeSpace(process.cwd())?.freeBytes).toBeGreaterThan(0) + expect(volumeSpace('/definitely/not/a/path/on/this/machine')).toBeNull() + }) +}) + +describe('the variables', () => { + it('have defaults that protect a deployment which sets none of them', () => { + expect(defaults).toMatchObject({ + guardOn: true, + beeMemoryBytes: 1024 ** 3, + memoryHeadroomPercent: 10, + warmupSeconds: 600, + beeDiskBytes: 2 * 1024 ** 3, + diskHeadroomPercent: 10, + memoryLimitBytes: null, + notes: [], + }) + }) + + it('replace an unusable value by the default and SAY SO', () => { + for (const bad of ['abc', '0', '-5', '999999999', '1.5']) { + const settings = resourceSettings({ TRIOS_QUEEN_BEE_MEMORY_MB: bad }) + expect(settings.beeMemoryBytes).toBe(1024 ** 3) + expect(settings.notes.join(' ')).toContain('TRIOS_QUEEN_BEE_MEMORY_MB') + } + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'maybe' }), + ).toMatchObject({ guardOn: true }) + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'maybe' }).notes, + ).toHaveLength(1) + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'OFF' }).guardOn, + ).toBe(false) + }) + + it('never tell an operator the container limit is 0', () => { + for (const bad of ['24GB', '256', '-1', '1.5']) { + const settings = resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: bad }) + expect(settings.memoryLimitBytes).toBeNull() + expect(settings.notes).toHaveLength(1) + expect(settings.notes[0]).toContain( + 'using the limit the container reports', + ) + expect(settings.notes[0]).not.toContain('using 0') + } + // Unset, empty and 0 are three spellings of "not stated": no note at all. + for (const unset of [undefined, '', ' ', '0']) { + const settings = resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: unset }) + expect(settings.memoryLimitBytes).toBeNull() + expect(settings.notes).toHaveLength(0) + } + expect( + resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }) + .memoryLimitBytes, + ).toBe(24_576 * 1024 * 1024) + }) + + it('are read on every call, so moving the line needs no restart', () => { + const env: NodeJS.ProcessEnv = {} + expect(resourceSettings(env).beeMemoryBytes).toBe(1024 ** 3) + env.TRIOS_QUEEN_BEE_MEMORY_MB = '2048' + expect(resourceSettings(env).beeMemoryBytes).toBe(2 * 1024 ** 3) + }) +}) + +describe('does one more bee fit', () => { + const judge = (memory: MemoryReading, youngBees = 0, volume = roomy) => + judgeBeeRoom({ + memory, + volume, + youngBees, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + }) + + it('admits a bee at the measured sixteen-bee load and refuses near the limit', () => { + expect(judge(measured(14.7))).toEqual({ ok: true }) + const refused = judge(measured(20.9)) + expect(refused.ok).toBe(false) + if (refused.ok) return + expect(refused.resource).toBe('memory') + expect(refused.detail).toMatch( + /^container memory is 20\.9 GB of 24\.0 GB in use \(cgroup v2\);/, + ) + expect(refused.detail).toContain('needs about 1.1 GB and 2.4 GB stays free') + expect(refused.detail.endsWith('.')).toBe(true) + // The report quotes `summary`: it is the first sentence of the detail, it + // carries every number, it names no variable and it fits a report line. + expect(refused.detail.startsWith(`${refused.summary}. `)).toBe(true) + expect(refused.summary).toContain('20.9 GB of 24.0 GB') + expect(refused.summary).toContain('1.1 GB and 2.4 GB stays free') + expect(refused.summary).not.toContain('TRIOS_') + expect(refused.summary.length).toBeLessThanOrEqual(200) + }) + + it('reserves for bees that started and have not grown, and says how many', () => { + const refused = judge(measured(10), 12) + expect(refused.ok).toBe(false) + if (refused.ok) return + expect(refused.detail).toContain( + '12 bee(s) started in the last 10 min are still growing', + ) + expect( + judge(measured(20.9)).ok === false && !judge(measured(20.9)).ok, + ).toBe(true) + const calm = judge(measured(20.9)) + if (!calm.ok) expect(calm.detail).not.toContain('still growing') + }) + + it('stops a round of back-to-back starts that all read the same empty container', () => { + // The reading does not move: a bee takes minutes to allocate what it will + // hold. Thirty-four starts would each be admitted on the reading alone. + let started = 0 + const nowMs = 1_000_000 + while (started < 34) { + const verdict = judgeBeeRoom({ + memory: measured(1), + volume: roomy, + youngBees: youngBeeCount(nowMs, defaults.warmupSeconds), + settings: defaults, + volumeDir: '/workspace', + }) + if (!verdict.ok) break + noteBeeStarted(`turn-${started}`, nowMs) + started += 1 + } + // (24 - 2.4 - 1) GB over 1024 MiB a bee. + expect(started).toBe(19) + }) + + it('forgets a start once the bee has had time to grow', () => { + noteBeeStarted('a', 0) + noteBeeStarted('b', 1_000) + noteBeeStarted('c', 700_000) + expect(youngBeeCount(701_000, 600)).toBe(1) + expect(youngBeeCount(701_000, 0)).toBe(0) + // A count never destroys what a later count needs: the window is read on + // every call and may have grown since. + expect(youngBeeCount(701_000, 3600)).toBe(3) + }) + + it('frees the reservation when the bee ends, however young it was', () => { + // `turn.ok` only means /chat answered 200; a provider refusal arrives as a + // 200 whose stream then fails. Ten such dead starts in ten minutes used to + // reserve ten GB for bees that no longer existed. + for (let i = 0; i < 18; i++) noteBeeStarted(`dead-${i}`, 1_000) + expect(judge(measured(1.9), youngBeeCount(2_000, 600)).ok).toBe(false) + for (let i = 0; i < 18; i++) noteBeeEnded(`dead-${i}`) + expect(youngBeeCount(2_000, 600)).toBe(0) + expect(judge(measured(1.9), youngBeeCount(2_000, 600))).toEqual({ + ok: true, + }) + // Ending a turn nobody noted is not an error. + noteBeeEnded('never-started') + }) + + it('names the window as it was set, not rounded to minutes', () => { + const at = (warmupSeconds: number) => { + const refused = judgeBeeRoom({ + memory: measured(10), + volume: roomy, + youngBees: 12, + settings: { ...defaults, warmupSeconds }, + volumeDir: '/workspace', + }) + return refused.ok ? '' : refused.summary + } + expect(at(20)).toContain('started in the last 20 s are') + expect(at(90)).toContain('started in the last 90 s are') + expect(at(600)).toContain('started in the last 10 min are') + }) + + it("refuses on disk with the reaper's counts in the sentence", () => { + const short = { totalBytes: 50 * GB, freeBytes: 5.1 * GB } + const first = judge(measured(3), 0, short) + expect(first.ok).toBe(false) + if (first.ok) return + expect(first.resource).toBe('disk') + expect(first.detail).toContain( + 'has 5.1 GB free of 50.0 GB (nothing reaped yet)', + ) + const after = judgeBeeRoom({ + memory: measured(3), + volume: short, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + reaped: { removed: 2, keptDirty: 1, keptRunning: 14 }, + }) + if (after.ok) throw new Error('expected a refusal') + expect(after.detail).toContain( + 'after a reap (2 removed; kept 1 with uncommitted work and 14 of running bees)', + ) + expect(after.detail).toContain('needs about 2.1 GB and 5.0 GB stays free') + // Where the reaper may stop: 2048 MiB + 5 GB of 50 GB is 85.7% used. + expect(diskLineUsedPercent(short, defaults)).toBe(85) + // A path may hold ". ", so the report reads the field and never the prose. + const dotted = judgeBeeRoom({ + memory: measured(3), + volume: short, + youngBees: 0, + settings: defaults, + volumeDir: '/Volumes/Ext. Drive/ws/BrowserOS', + }) + if (dotted.ok) throw new Error('expected a refusal') + expect(dotted.summary).toContain( + '/Volumes/Ext. Drive/ws/BrowserOS has 5.1 GB', + ) + expect(dotted.summary.endsWith('GB stays free')).toBe(true) + }) + + it('leaves a checkout that is not there to the code that can say so', () => { + // The entrypoint starts the server even when the clone failed. `df` on the + // missing directory gives nothing, and "no room in the container: disk" + // with nineteen GB free hid the git error that names the real cause. + expect( + judgeBeeRoom({ + memory: measured(3), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + checkoutMissing: true, + }), + ).toEqual({ ok: true }) + // Memory is still judged: a full container is full wherever the checkout is. + expect( + judgeBeeRoom({ + memory: measured(23), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + checkoutMissing: true, + }).ok, + ).toBe(false) + }) + + it('does not let a share of a huge disk refuse a laptop with ninety GB to spare', () => { + const laptop = { totalBytes: 1000 * GB, freeBytes: 90 * GB } + expect( + judge({ kind: 'unsupported', platform: 'darwin' }, 0, laptop), + ).toEqual({ ok: true }) + }) + + it('treats unknown as a refusal and not-Linux as nothing to protect', () => { + const blind = judge({ + kind: 'unknown', + tried: ['cgroup v2', 'cgroup v1', '/proc/meminfo'], + }) + if (blind.ok) throw new Error('expected a refusal') + expect(blind.detail).toContain('unknown is not room') + expect(blind.detail).toContain('TRIOS_QUEEN_RESOURCE_GUARD=off') + const noDisk = judgeBeeRoom({ + memory: measured(3), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace', + }) + if (noDisk.ok) throw new Error('expected a refusal') + expect(noDisk.resource).toBe('disk') + expect(judge({ kind: 'unsupported', platform: 'darwin' })).toEqual({ + ok: true, + }) + }) + + it('reports one cause, memory first, and leaks nothing of the closed breakdown', () => { + const both = judge(measured(23.5), 0, { + totalBytes: 50 * GB, + freeBytes: 1 * GB, + }) + if (both.ok) throw new Error('expected a refusal') + expect(both.resource).toBe('memory') + for (const word of [ + 'connectedCredentials', + 'lanesPerCredential', + 'effectiveCapacity', + 'API_KEY', + ]) { + expect(both.detail).not.toContain(word) + } + }) +}) diff --git a/trios/agent-server/apps/server/tests/api/queen-round.test.ts b/trios/agent-server/apps/server/tests/api/queen-round.test.ts index 62134ff6cd..11c8be6410 100644 --- a/trios/agent-server/apps/server/tests/api/queen-round.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-round.test.ts @@ -1,6 +1,12 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import type { Pool } from 'pg' @@ -137,9 +143,20 @@ function roundPool(finished: FinishedRow[] = []) { const saved: Record = {} const realFetch = globalThis.fetch +/** What the one case that connects a credential sets, so no later case inherits it. */ +const WORKER_POOL_KEYS = [ + 'TRIOS_QUEEN_WORKER_PROVIDER', + 'TRIOS_QUEEN_WORKER_BASE_URL', + 'TRIOS_QUEEN_WORKER_API_KEY', + 'TRIOS_QUEEN_RESOURCE_GUARD', + 'TRIOS_QUEEN_BEE_MEMORY_MB', + 'TRIOS_QUEEN_BEE_DISK_MB', +] + beforeEach(() => { for (const key of [ ...PROVIDER_KEYS, + ...WORKER_POOL_KEYS, QUEEND_ENV, 'WORKSPACE_DIR', 'TRIOS_GITHUB_REPO', @@ -257,6 +274,54 @@ describe('queen round, lease lost', () => { ) }) +/** + * A round the policy ALLOWED and the container refused. + * + * `queend` says yes, a credential is free, and the container has no room. The + * report used to read "Started nothing. No reason given." under the headline + * "nothing to do" - with the guard's sentence one line lower, in the "Refused" + * line, where a headline reader never looks. The pure helpers are pinned in + * queen-report-lines.test.ts; this pins that the round actually calls them, which + * a review showed no test did: both call sites could be reverted to + * `choice.refusal` with every suite green. + */ +describe('queen round, no room in the container', () => { + it.if(present)( + 'says so in the report, with a label in the headline and the sentence in the body', + async () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'https://api.z.ai/api/paas/v4' + process.env.TRIOS_QUEEN_WORKER_API_KEY = 'a' + // No test machine has room for a 64 GiB bee holding a 1 TiB worktree. On + // Linux the refusal is memory; where memory is not measured it is disk - + // measured on a workspace that EXISTS, because a missing one is left for + // `prepareWorktree` to report. Either way the container refuses. + process.env.TRIOS_QUEEN_BEE_MEMORY_MB = '65536' + process.env.TRIOS_QUEEN_BEE_DISK_MB = '1048576' + process.env.WORKSPACE_DIR = realpathSync( + mkdtempSync(join(tmpdir(), 'queen-round-no-room-')), + ) + mkdirSync(join(process.env.WORKSPACE_DIR, 'BrowserOS')) + const { pool, queries } = roundPool() + await runRound(pool, 'me', 7, { held: true }, [ISSUE]) + + // Nothing is booked against the issue: the refusal is about the + // container, and it is how nearly every memory-bound round ends. + expect(queries.some((q) => isDispatchInsert(q.sql))).toBe(false) + const report = queries.find((q) => + q.sql.includes('INSERT INTO queen_report'), + ) + expect(report).toBeDefined() + const [headline, body] = report?.params as [string, string] + expect(headline).toMatch(/^no room in the container: (memory|disk)$/) + expect(body).not.toContain('No reason given') + expect(body).toMatch( + /Started nothing\. (container memory|free space on|volume) .+\. \d+ issue\(s\) were on the table\./, + ) + }, + ) +}) + /** * The send-back count, end to end, through the real policy. * diff --git a/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts b/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts index 89c27a95bf..6c62a38d38 100644 --- a/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts @@ -1,5 +1,18 @@ import { describe, expect, it } from 'bun:test' -import { volumeUsedPercent } from '../../src/api/services/queen-dispatch' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + prepareWorktree, + reapWorktrees, + volumeUsedPercent, +} from '../../src/api/services/queen-dispatch' // WHY THIS EXISTS. // @@ -12,10 +25,12 @@ import { volumeUsedPercent } from '../../src/api/services/queen-dispatch' // while the application is unhealthy - which it was, BECAUSE the volume was // full. The collector depended on the thing whose failure it collects for. // -// These pin the properties that make the in-container collector safe. The -// removal path itself needs a real filesystem and a real git, so it is exercised -// in production behind the watermark rather than mocked into a shape that would -// agree with whatever it was written against. +// These pin the properties that make the in-container collector safe. Most of +// them read the source, because the removal path needs a real filesystem and a +// real git. The one property a reading could not pin - WHOSE tree is removed - +// runs against a real scratch repository at the bottom of this file: a review +// deleted the `continue` after `keptRunning.push` and every text assertion +// stayed green while a running bee's tree was removed and reported as kept. describe('volumeUsedPercent', () => { it('measures a directory that exists', () => { @@ -29,13 +44,16 @@ describe('volumeUsedPercent', () => { // UNKNOWN IS NOT ROOM. A guard that reads an unmeasurable disk as empty // disables itself exactly when the filesystem is unwell, which is the one // moment it is needed. - expect(volumeUsedPercent('/definitely/not/a/path/on/this/machine')).toBeNull() + expect( + volumeUsedPercent('/definitely/not/a/path/on/this/machine'), + ).toBeNull() }) }) describe('the reaper refuses the things that would lose work', () => { const source = Bun.file( - new URL('../../src/api/services/queen-dispatch.ts', import.meta.url).pathname, + new URL('../../src/api/services/queen-dispatch.ts', import.meta.url) + .pathname, ) it('never removes a worktree holding uncommitted work', async () => { @@ -46,7 +64,21 @@ describe('the reaper refuses the things that would lose work', () => { const fn = text.slice(text.indexOf('export async function reapWorktrees')) const body = fn.slice(0, fn.indexOf('\n}\n')) expect(body).toContain('keptDirty') - expect(body).toMatch(/dirty\.code !== 0 \|\| dirty\.out\.trim\(\)\.length > 0/) + expect(body).toMatch( + /dirty\.code !== 0 \|\| dirty\.out\.trim\(\)\.length > 0/, + ) + }) + + it('asks whether a bee is running BEFORE it asks whether the tree is dirty', async () => { + // A running bee's tree is kept whatever its status reads, and the status + // call is a subprocess per tree that a protected tree should not cost. + const text = await source.text() + const fn = text.slice(text.indexOf('export async function reapWorktrees')) + const body = fn.slice(0, fn.indexOf('\n}\n')) + expect(body.indexOf('opts.protect?.has(')).toBeGreaterThan(-1) + expect(body.indexOf('opts.protect?.has(')).toBeLessThan( + body.indexOf("['status', '--porcelain']"), + ) }) it('treats an unreadable status as dirty, not as clean', async () => { @@ -81,12 +113,182 @@ describe('the dispatch refuses before it dies half-way', () => { // The dispatch that exposed this died with "cannot update the ref ... unable // to write file", which sent every reader looking at git rather than at df. const text = await Bun.file( - new URL('../../src/api/services/queen-dispatch.ts', import.meta.url).pathname, + new URL('../../src/api/services/queen-dispatch.ts', import.meta.url) + .pathname, ).text() - const prep = text.slice(text.indexOf('export async function prepareWorktree')) + const prep = text.slice( + text.indexOf('export async function prepareWorktree'), + ) const beforeFetch = prep.slice(0, prep.indexOf("['fetch'")) expect(beforeFetch).toContain('volumeUsedPercent') expect(beforeFetch).toContain('reapWorktrees') expect(beforeFetch).toMatch(/volume \$\{still\}% full after reaping/) }) }) + +/** + * A real repository with real worktrees, because "whose tree was removed" is a + * fact about a filesystem. `origin.git` is bare, `BrowserOS` is the Queen's + * checkout - the name `workspaceRoot()` derives - and every name given becomes + * a clean worktree under `.worktrees/`, which is where dispatch cuts them. + */ +function scratchRepository(branches: string[]): { + root: string + tree: (branch: string) => string + restore: () => void +} { + const REAL_GIT = Bun.which('git') || '/usr/bin/git' + const git = (cwd: string, args: string[]): void => { + const done = Bun.spawnSync([REAL_GIT, ...args], { + cwd, + env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' }, + }) + if (done.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${done.stderr.toString()}`) + } + } + const previous = { + WORKSPACE_DIR: process.env.WORKSPACE_DIR, + TRIOS_REPO_REF: process.env.TRIOS_REPO_REF, + TRIOS_REPO_URL: process.env.TRIOS_REPO_URL, + QUEEN_VOLUME_KEEP: process.env.QUEEN_VOLUME_KEEP, + // The marks the cases below are written against (80 and 55). Left as the + // shell has them, QUEEN_VOLUME_HIGH=90 makes "85% reaps" a lie. + QUEEN_VOLUME_HIGH: process.env.QUEEN_VOLUME_HIGH, + QUEEN_VOLUME_LOW: process.env.QUEEN_VOLUME_LOW, + } + const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'queen-reap-'))) + const origin = join(scratch, 'origin.git') + git(scratch, ['init', '-q', '--bare', 'origin.git']) + const seed = join(scratch, 'seed') + mkdirSync(seed) + git(seed, ['init', '-q', '-b', 'dev']) + git(seed, ['config', 'user.email', 'bee@example.com']) + git(seed, ['config', 'user.name', 'Bee']) + writeFileSync(join(seed, 'README.md'), 'one good branch\n') + git(seed, ['add', '.']) + git(seed, ['-c', 'commit.gpgsign=false', 'commit', '-qm', 'first']) + git(seed, ['push', '-q', origin, 'dev']) + git(origin, ['symbolic-ref', 'HEAD', 'refs/heads/dev']) + const root = join(scratch, 'BrowserOS') + git(scratch, ['clone', '-q', origin, 'BrowserOS']) + const tree = (branch: string) => join(root, '.worktrees', branch) + for (const branch of branches) { + git(root, [ + 'worktree', + 'add', + '-q', + '-b', + branch, + tree(branch), + 'origin/dev', + ]) + } + process.env.WORKSPACE_DIR = scratch + process.env.TRIOS_REPO_REF = 'origin/dev' + delete process.env.TRIOS_REPO_URL + // The default keeps the newest six whoever they belong to, which in a + // repository of three trees is all of them: nothing would be a candidate and + // the test would pass with the protection deleted. + process.env.QUEEN_VOLUME_KEEP = '0' + delete process.env.QUEEN_VOLUME_HIGH + delete process.env.QUEEN_VOLUME_LOW + return { + root, + tree, + restore: () => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }, + } +} + +describe('the reaper, on a real repository', () => { + it('keeps the clean tree of a running bee and removes the one nobody is running', async () => { + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + const gc = await reapWorktrees({ + high: 0, + low: 0, + volumeUsed: () => 90, + protect: new Set(['queen-7']), + }) + expect(gc.keptRunning).toEqual([repo.tree('queen-7')]) + expect(gc.removed).toEqual([repo.tree('queen-3')]) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(false) + } finally { + repo.restore() + } + }) + + it('protects them in the reap prepareWorktree runs, which fires before the container guard does', async () => { + // 85% used: above QUEEN_VOLUME_HIGH, and on a 50 GB volume still 7.5 GB + // free - room as far as the guard is concerned. This reap was the + // unprotected one: two running bees lost their trees to it in a scratch + // reproduction before `running` existed. + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + let asked = 0 + const prepared = await prepareWorktree(99, { + volumeUsed: () => 85, + running: async () => { + asked += 1 + return new Set(['queen-7']) + }, + }) + expect(prepared.ok).toBe(true) + expect(asked).toBe(1) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(false) + expect(existsSync(repo.tree('queen-99'))).toBe(true) + } finally { + repo.restore() + } + }) + + it('reaps nothing when it cannot find out who is running', async () => { + // Null is "the registry could not be read". It used to arrive as an empty + // set - "nobody is running" - and the reap went ahead unprotected. + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + const prepared = await prepareWorktree(97, { + volumeUsed: () => 85, + running: async () => null, + }) + expect(prepared.ok).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(true) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + // And a volume that is truly full still refuses, saying why nothing went. + const full = await prepareWorktree(96, { + volumeUsed: () => 97, + running: async () => null, + }) + expect(full.ok).toBe(false) + expect(full.detail).toContain('volume 97% full and nothing was reaped') + } finally { + repo.restore() + } + }) + + it('does not ask who is running when there is nothing to reap', async () => { + const repo = scratchRepository(['queen-3']) + try { + let asked = 0 + const prepared = await prepareWorktree(98, { + volumeUsed: () => 40, + running: async () => { + asked += 1 + return new Set() + }, + }) + expect(prepared.ok).toBe(true) + expect(asked).toBe(0) + expect(existsSync(repo.tree('queen-3'))).toBe(true) + } finally { + repo.restore() + } + }) +}) From 659328b75bc10007ac75760c2293ebe12696c98e Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Fri, 18 Sep 2026 02:21:37 +0700 Subject: [PATCH 2/2] fix(queen): a bee's session outlived its turn, so the container filled with agents nobody was running MEASURED ON THE DEPLOYED CONTAINER, 2026-09-18, from the Railway metrics. Memory did not follow the bees; it followed the ENDINGS: 01:41 3 bees 1.8 GB 02:00 3 bees 10.5 GB 01:47 5 bees 2.0 GB 02:02 2 bees 11.1 GB 01:53 6 bees 2.5 GB 02:06 1 bee 11.0 GB 01:55 5 bees 5.0 GB 02:10 0 bees 11.0 GB Seven bees cost 2.5 GB between them. An hour later, with no bee running at all, the container held eleven and gave none of it back. An earlier round the same night reached 21.9 GB of 24 and was saved by a restart. Nothing here was about how many bees run at once, which is why the ceiling the operator's watchdog kept cutting - eighteen bees to nine - bought nothing: it was charging the bees for memory that was never theirs. Every dispatch gets a fresh conversation id, `/chat` builds an `AgentSession` for it and keeps it in the SessionStore, and nothing in this file ever asked for it back. The session holds the agent with the whole turn's message history; for a 65k-context coding turn that is more than a gigabyte. So the baseline grew with every bee that FINISHED, and only a redeploy cleared it. `DELETE /chat/:conversationId` disposes the agent and drops it from the store. It has existed all along, for the UI. `closeDispatch` now calls it, over the same loopback the dispatch already uses to start a turn, before the refill signal - so the memory is back before the next bee is started against it. A turn that is over needs none of it: the review reads the transcript from `queen_transcript`, and a bee's conversation id is used once and never again. Failing to release is logged and nothing more; the ending stands either way. Co-Authored-By: Claude Opus 5 --- .../server/src/api/services/queen-dispatch.ts | 57 +++++++++++++++++++ .../server/tests/api/queen-dispatch.test.ts | 52 +++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 6bf018529e..291e319178 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -2672,6 +2672,9 @@ export async function closeDispatch( // The stream is over, so what the container guard reserved for this bee while // it was young is over too - whatever the database does below. noteBeeEnded(conversationId) + // ...and so is the agent that ran it. Released BEFORE the refill signal at + // the end, so the memory is back before the next bee is started against it. + await releaseSession(conversationId) // Whether the row reads finished on the database when this returns. That is // the ONLY condition under which the slot may be announced as free: a signal // about a row that still says `running` wakes a round that sees the bee as @@ -3186,6 +3189,60 @@ export async function runningBeeBranches( } } +/** + * Give back the agent that ran this turn. + * + * THE MEASUREMENT, 2026-09-18, from the Railway metrics of the deployed + * container. Memory did not follow the bees; it followed the ENDINGS: + * + * 01:41 3 bees 1.8 GB 02:00 3 bees 10.5 GB + * 01:47 5 bees 2.0 GB 02:02 2 bees 11.1 GB + * 01:53 6 bees 2.5 GB 02:06 1 bee 11.0 GB + * 01:55 5 bees 5.0 GB 02:10 0 bees 11.0 GB + * + * Seven bees cost 2.5 GB between them; an hour later, with NO bee running, the + * container held eleven and never gave a byte of it back. An earlier round the + * same night reached 21.9 GB of 24 and was only saved by a restart. The swarm's + * ceiling had been cut from 18 to 9 by the site's watchdog, which could see the + * memory but not the cause and charged it to the bees, where it never was. + * + * The cause is here: every dispatch gets a fresh conversation id, `/chat` + * builds an `AgentSession` for it and keeps it in the SessionStore, and nothing + * in this file ever asked for it back. The session holds the agent with the + * whole turn's message history - for a 65k-context coding turn, more than a + * gigabyte - so the baseline grew with every bee that FINISHED and only a + * redeploy cleared it. `DELETE /chat/:conversationId` disposes the agent and + * drops it from the store; it has existed all along, for the UI. + * + * A turn that is over needs none of it: the review reads the transcript from + * `queen_transcript`, and a bee's conversation id is used once and never again. + * Failing to release is logged and nothing more - the ending stands either way. + */ +async function releaseSession(conversationId: string): Promise { + const token = process.env.TRIOS_API_TOKEN + if (!token) return + const port = process.env.PORT || '8080' + try { + const response = await fetch( + `http://127.0.0.1:${port}/chat/${conversationId}`, + { method: 'DELETE', headers: { Authorization: `Bearer ${token}` } }, + ) + // 404 is the honest answer for a turn whose session was never built, and + // means the same thing as a delete: nothing is held any more. + if (!response.ok && response.status !== 404) { + logger.warn('Queen could not release the session of a finished bee', { + conversationId, + status: response.status, + }) + } + } catch (error) { + logger.warn('Queen could not release the session of a finished bee', { + conversationId, + error: error instanceof Error ? error.message : String(error), + }) + } +} + export async function recordDispatch( pool: Pool, issue: number, diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index 94e9a1d841..20cbae43c1 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -1604,6 +1604,58 @@ describe('draining a turn', () => { ).toBe(true) }) + it('gives the agent back when the turn is over, so the memory comes with it', async () => { + // Measured on the deployed container 2026-09-18: seven bees cost 2.5 GB + // between them, and an hour later, with none running, it held 11 GB and + // never gave it back. Every dispatch builds a session; nothing asked for + // one back, so the baseline grew with every bee that FINISHED. + const { pool } = recordingPool() + const asked: Array<{ url: string; method?: string; auth?: string }> = [] + const realFetch = globalThis.fetch + const previous = process.env.TRIOS_API_TOKEN + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + globalThis.fetch = (async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + asked.push({ + url: String(url), + method: init?.method, + auth: new Headers(init?.headers).get('Authorization') ?? undefined, + }) + return new Response('{}', { status: 200 }) + }) as unknown as typeof fetch + try { + await drain(pool, sse([{ type: 'finish' }]), 'the-turn-that-ended', 1244) + } finally { + globalThis.fetch = realFetch + if (previous === undefined) delete process.env.TRIOS_API_TOKEN + else process.env.TRIOS_API_TOKEN = previous + } + expect(asked).toHaveLength(1) + expect(asked[0].method).toBe('DELETE') + expect(asked[0].url).toEndWith('/chat/the-turn-that-ended') + expect(asked[0].auth).toBe('Bearer a-token-for-the-test') + }) + + it('still closes the dispatch when the session will not be released', async () => { + const { pool, asked } = recordingPool() + const realFetch = globalThis.fetch + const previous = process.env.TRIOS_API_TOKEN + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + globalThis.fetch = (async () => { + throw new Error('connection refused') + }) as unknown as typeof fetch + try { + await drain(pool, sse([{ type: 'finish' }]), 'conv-9', 1244) + } finally { + globalThis.fetch = realFetch + if (previous === undefined) delete process.env.TRIOS_API_TOKEN + else process.env.TRIOS_API_TOKEN = previous + } + expect(touching(asked, 'UPDATE queen_dispatch')).toHaveLength(1) + }) + it('reports no price rather than a free turn when the frame never came', async () => { const { pool, asked } = recordingPool() await drain(pool, sse([{ type: 'text-delta', delta: 'killed' }]), 'c', 1244)