diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 3423171b33..291e319178 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -29,6 +29,19 @@ import { statSync } from 'node:fs' import type { Pool } from 'pg' import { logger } from '../../lib/logger' import { shellArgv } from '../../tools/filesystem/bash' +import { + describeReading, + diskLineUsedPercent, + judgeBeeRoom, + type MemoryReading, + noteBeeEnded, + noteBeeStarted, + readContainerMemory, + resourceSettings, + type VolumeSpace, + volumeSpace, + youngBeeCount, +} from './queen-resources' import { workerSystemPrompt } from './queen-tick' /** @@ -1673,12 +1686,22 @@ export async function reapWorktrees( low?: number keepNewest?: number volumeUsed?: (dir: string) => number | null + /** + * Branch names (`queen-`) of bees that are running right now. + * + * "The newest few are likely to be running" was true while four bees ran. + * With twenty, the seventh-newest tree belongs to a bee too, and a bee that + * has read a lot and written nothing yet has a CLEAN tree - exactly what + * this function removes. Its work would vanish under it mid-turn. + */ + protect?: ReadonlySet } = {}, ): Promise<{ before: number | null after: number | null removed: string[] keptDirty: string[] + keptRunning: string[] refused: string[] }> { const high = opts.high ?? Number(process.env.QUEEN_VOLUME_HIGH ?? 80) @@ -1693,6 +1716,7 @@ export async function reapWorktrees( after: before, removed: [] as string[], keptDirty: [] as string[], + keptRunning: [] as string[], refused: [] as string[], } // Unknown is not room. Below the mark is not an emergency. @@ -1722,6 +1746,10 @@ export async function reapWorktrees( const now = measure(root) if (now !== null && now <= low) break + if (opts.protect?.has(c.path.slice(c.path.lastIndexOf('/') + 1))) { + result.keptRunning.push(c.path) + continue + } const dirty = await run('git', ['status', '--porcelain'], c.path, 60_000) // Unreadable is not clean. A tree whose state cannot be read might hold the // only copy of a turn's work. @@ -1863,6 +1891,18 @@ export async function prepareWorktree( // filesystem is a guard no test can pin. So the measurement is a dependency // with a real default, like the pool and the clock everywhere else here. volumeUsed?: (dir: string) => number | null + /** + * The branches of bees running right now, asked for only when a reap is + * about to happen. + * + * Without it this reap took the clean tree of a running bee. It fires at + * QUEEN_VOLUME_HIGH - 80% used - which on a 50 GB volume is EARLIER than the + * container guard refuses (7 GB free is 86%), so between the two marks the + * guard saw room, reaped nothing, and this call reaped unprotected. + * Reproduced on a scratch repository before it was fixed: ten clean trees, + * two of them registered as running, both gone. + */ + running?: () => Promise | null> } = {}, ): Promise<{ ok: boolean; path: string; detail: string }> { const measure = deps.volumeUsed ?? volumeUsedPercent @@ -1972,15 +2012,24 @@ export async function prepareWorktree( const used = measure(root) const highMark = Number(process.env.QUEEN_VOLUME_HIGH ?? 80) if (used !== null && used >= highMark) { - const gc = await reapWorktrees({ volumeUsed: measure }) - logger.warn('Queen reaped worktrees before cutting a new one', { - before: gc.before, - after: gc.after, - removed: gc.removed.length, - keptDirty: gc.keptDirty.length, - refused: gc.refused.length, - }) - const still = gc.after + // Null is "the registry could not be read": nothing is reaped then, and the + // refusal below still stands between a full volume and a half-cut tree. + const protect = deps.running ? await deps.running() : undefined + const gc = + protect === null + ? null + : await reapWorktrees({ volumeUsed: measure, protect }) + if (gc) { + logger.warn('Queen reaped worktrees before cutting a new one', { + before: gc.before, + after: gc.after, + removed: gc.removed.length, + keptDirty: gc.keptDirty.length, + keptRunning: gc.keptRunning.length, + refused: gc.refused.length, + }) + } + const still = gc ? gc.after : used if (still !== null && still >= 95) { // REFUSE, and say what is true. Dying at `git worktree add` reports a git // error for a disk problem, and every reader of that message has looked in @@ -1989,10 +2038,13 @@ export async function prepareWorktree( return { ok: false, path, - detail: - `volume ${still}% full after reaping ${gc.removed.length} worktree(s); ` + - `${gc.keptDirty.length} held uncommitted work and were kept. ` + - 'Not cutting a worktree that would fail part-way', + detail: gc + ? `volume ${still}% full after reaping ${gc.removed.length} worktree(s); ` + + `${gc.keptDirty.length} held uncommitted work and ` + + `${gc.keptRunning.length} belong to running bees and were kept. ` + + 'Not cutting a worktree that would fail part-way' + : `volume ${still}% full and nothing was reaped, because the running ` + + 'bees could not be listed. Not cutting a worktree that would fail part-way', } } } @@ -2617,6 +2669,12 @@ export async function closeDispatch( outcome: string, tokens?: TokenUsage, ): Promise { + // The stream is over, so what the container guard reserved for this bee while + // it was young is over too - whatever the database does below. + noteBeeEnded(conversationId) + // ...and so is the agent that ran it. Released BEFORE the refill signal at + // the end, so the memory is back before the next bee is started against it. + await releaseSession(conversationId) // Whether the row reads finished on the database when this returns. That is // the ONLY condition under which the slot may be announced as free: a signal // about a row that still says `running` wakes a round that sees the bee as @@ -2792,6 +2850,13 @@ export interface DispatchOutcome { conversationId?: string /** Which provider key this bee took, so the next one takes a different one. */ keyIndex?: number + /** + * Set only when the CONTAINER refused the start. The report reads it to tell + * this refusal from every other one, because this is the only one whose + * sentence it may quote outside the "Refused #N" line - and it quotes + * `summary`, never a piece cut out of `detail`. + */ + room?: { resource: 'memory' | 'disk'; summary: string } } /** @@ -2814,6 +2879,8 @@ export async function dispatchBee( */ criteria: string[] = [], criteriaSource = 'none', + /** Measurements, injectable so a test can put the container in any state. */ + deps: BeeRoomDeps = {}, ): Promise { const branch = `queen-${issue}` @@ -2856,7 +2923,37 @@ export async function dispatchBee( return { started: false, issue, branch, detail } } - const worktree = await prepareWorktree(issue) + // A key is free and a provider answers. The remaining question is the one + // nothing used to ask: can the CONTAINER carry another bee? Asked here, before + // a worktree is cut, so a refusal costs a measurement and nothing else. The + // tick loop breaks on a refused dispatch and the next round fires when a bee + // finishes - which is exactly when memory comes back. + const noRoom = await beeRoomRefusal(pool, branch, deps) + if (noRoom) { + logger.warn( + 'Queen tick chose an issue but the container cannot carry another bee', + { issue, resource: noRoom.resource, detail: noRoom.detail }, + ) + // NOT RECORDED AGAINST THE ISSUE, unlike every refusal above. Those say + // something about the dispatch; this says something about the container, + // and once the swarm is memory-bound it is how nearly every round ends. + // `recordDispatch` would archive a history row and rewrite the issue's live + // row each time - wiping the last attempt's conversation, tokens and + // outcome for a reason that has nothing to do with the issue. The warning + // above and the round report, one row per round already, carry it. + return { + started: false, + issue, + branch, + detail: noRoom.detail, + room: { resource: noRoom.resource, summary: noRoom.summary }, + } + } + + const worktree = await prepareWorktree(issue, { + running: () => runningBeeBranches(pool), + volumeUsed: deps.volumeUsed, + }) if (!worktree.ok) { await recordDispatch( pool, @@ -2925,6 +3022,9 @@ export async function dispatchBee( chosen.provider, chosen.model, ) + // A bee that just started has not allocated what it will hold. The next + // dispatch of this round must not read the container as empty because of it. + if (turn.ok) noteBeeStarted(conversationId, (deps.now ?? Date.now)()) // ONLY NOW may the stream be read. Everything that reads the bee's output // eventually writes to the row above, and a writer that can outrun the row's // creation is a writer that silently updates nothing. @@ -2940,6 +3040,209 @@ export async function dispatchBee( } } +export interface BeeRoomDeps { + memory?: () => MemoryReading + volume?: (dir: string) => VolumeSpace | null + reap?: typeof reapWorktrees + now?: () => number + /** Handed to `prepareWorktree`, whose own volume check reads the real disk. */ + volumeUsed?: (dir: string) => number | null +} + +/** + * The sentence that refuses a start because the container has no room, or null. + * + * Disk gets one chance to make room before it refuses, because it is the one + * resource this server can free itself: the reaper only ever ran inside + * `prepareWorktree`, which is AFTER this gate, so a gate that refused on a full + * volume would have stopped the only code that empties it. The reap protects + * the trees of running bees - see `reapWorktrees`. Memory gets no such chance: + * nothing here can free it except a bee finishing. + */ +async function beeRoomRefusal( + pool: Pool, + branch: string, + deps: BeeRoomDeps, +): Promise<{ + resource: 'memory' | 'disk' + summary: string + detail: string +} | null> { + const settings = resourceSettings() + if (!settings.guardOn) { + sayOncePerChange( + 'Queen resource guard is OFF by TRIOS_QUEEN_RESOURCE_GUARD; dispatching without checking memory or disk', + {}, + true, + ) + return null + } + const root = workspaceRoot() + const now = (deps.now ?? Date.now)() + const judge = (reaped?: { + removed: number + keptDirty: number + keptRunning: number + }) => { + const memory = (deps.memory ?? readContainerMemory)() + const volume = (deps.volume ?? volumeSpace)(root) + // Only the REAL measurement can mistake a missing checkout for a sick disk; + // an injected one answers for itself. + const checkoutMissing = !deps.volume && !pathExists(root) + const notes = [ + ...settings.notes, + ...(memory.kind === 'measured' && memory.note ? [memory.note] : []), + ] + // What it reads, once - so the first thing an operator checks after a + // deploy (is it looking at the container or at the host?) is in the log + // before anything is ever refused. + sayOncePerChange( + 'Queen resource guard is on', + { reads: describeReading(memory), volume: root, ignored: notes }, + notes.length > 0, + ) + const room = judgeBeeRoom({ + memory, + volume, + youngBees: youngBeeCount(now, settings.warmupSeconds), + settings, + volumeDir: root, + reaped, + checkoutMissing, + }) + return { room, volume } + } + const first = judge() + if (first.room.ok) return null + if (first.room.resource !== 'disk' || first.volume === null) return first.room + + // WHO IS RUNNING MUST BE KNOWN, or nothing is reaped. An unreadable registry + // used to mean "protect nobody", which is the unprotected reap this change + // exists to end; and skipping a reap refuses nothing the disk had not already + // refused. The next round asks again. + const running = await runningBeeBranches(pool) + if (running === null) return first.room + const gc = await (deps.reap ?? reapWorktrees)({ + // ...and the tree of the issue being dispatched, which is by definition not + // running. Without it a re-dispatch could reap its own tree here, and + // `prepareWorktree` would then cut it afresh with `-B`: the previous + // attempt's unpushed commits dropped off the branch. Reproduced on a + // scratch repository before this line existed. + protect: new Set([...running, branch]), + // `high: 0`: the gate has already established that the volume is short for + // THIS container, whatever percentage that happens to be. `low`: and it + // stops where the rule is satisfied, not at the collector's own low mark. + high: 0, + low: diskLineUsedPercent(first.volume, settings), + }) + const second = judge({ + removed: gc.removed.length, + keptDirty: gc.keptDirty.length, + keptRunning: gc.keptRunning.length, + }) + return second.room.ok ? null : second.room +} + +/** + * Said when it changes, not on every dispatch. The guard runs before every + * start; a line per start about a setting that has not moved is how a log + * teaches its reader to skip the line that matters. + */ +let guardLastSaid = '' +function sayOncePerChange( + message: string, + fields: Record, + warn: boolean, +): void { + const said = `${message} ${JSON.stringify(fields)}` + if (said === guardLastSaid) return + guardLastSaid = said + if (warn) logger.warn(message, fields) + else logger.info(message, fields) +} + +/** + * The branch names (`queen-`) of the bees the registry says are running, + * or NULL when the registry cannot be read. + * + * A worktree directory is named after its branch, so this is the set a reap + * must not touch however old and clean a tree looks. Null and not an empty set: + * "nobody is running" and "I could not find out" are different answers, and a + * caller that reaps on the second one removes the trees of running bees. + */ +export async function runningBeeBranches( + pool: Pool, +): Promise | null> { + try { + const rows = await pool.query( + `SELECT issue FROM queen_dispatch + WHERE started = true AND finished_at IS NULL`, + ) + return new Set( + (rows.rows ?? []).map((row: { issue: unknown }) => `queen-${row.issue}`), + ) + } catch (error) { + logger.warn('Queen could not list running bees, so nothing is reaped', { + error: error instanceof Error ? error.message : String(error), + }) + return null + } +} + +/** + * Give back the agent that ran this turn. + * + * THE MEASUREMENT, 2026-09-18, from the Railway metrics of the deployed + * container. Memory did not follow the bees; it followed the ENDINGS: + * + * 01:41 3 bees 1.8 GB 02:00 3 bees 10.5 GB + * 01:47 5 bees 2.0 GB 02:02 2 bees 11.1 GB + * 01:53 6 bees 2.5 GB 02:06 1 bee 11.0 GB + * 01:55 5 bees 5.0 GB 02:10 0 bees 11.0 GB + * + * Seven bees cost 2.5 GB between them; an hour later, with NO bee running, the + * container held eleven and never gave a byte of it back. An earlier round the + * same night reached 21.9 GB of 24 and was only saved by a restart. The swarm's + * ceiling had been cut from 18 to 9 by the site's watchdog, which could see the + * memory but not the cause and charged it to the bees, where it never was. + * + * The cause is here: every dispatch gets a fresh conversation id, `/chat` + * builds an `AgentSession` for it and keeps it in the SessionStore, and nothing + * in this file ever asked for it back. The session holds the agent with the + * whole turn's message history - for a 65k-context coding turn, more than a + * gigabyte - so the baseline grew with every bee that FINISHED and only a + * redeploy cleared it. `DELETE /chat/:conversationId` disposes the agent and + * drops it from the store; it has existed all along, for the UI. + * + * A turn that is over needs none of it: the review reads the transcript from + * `queen_transcript`, and a bee's conversation id is used once and never again. + * Failing to release is logged and nothing more - the ending stands either way. + */ +async function releaseSession(conversationId: string): Promise { + const token = process.env.TRIOS_API_TOKEN + if (!token) return + const port = process.env.PORT || '8080' + try { + const response = await fetch( + `http://127.0.0.1:${port}/chat/${conversationId}`, + { method: 'DELETE', headers: { Authorization: `Bearer ${token}` } }, + ) + // 404 is the honest answer for a turn whose session was never built, and + // means the same thing as a delete: nothing is held any more. + if (!response.ok && response.status !== 404) { + logger.warn('Queen could not release the session of a finished bee', { + conversationId, + status: response.status, + }) + } + } catch (error) { + logger.warn('Queen could not release the session of a finished bee', { + conversationId, + error: error instanceof Error ? error.message : String(error), + }) + } +} + export async function recordDispatch( pool: Pool, issue: number, diff --git a/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts b/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts index 74cfdcb44f..b2718ea75f 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-report-lines.ts @@ -26,6 +26,9 @@ export interface DispatchReportOutcome { started?: boolean issue?: number detail?: string + /** Set only when the CONTAINER refused the start: which resource had no + * room, and the guard's own one-sentence account of it. */ + room?: { resource: 'memory' | 'disk'; summary?: string } } /** @@ -111,6 +114,52 @@ export function refusedLines( }) } +/** + * Why a round that `queend` allowed still started nothing, when the reason is + * the container. + * + * `queend` refuses for the policy's reasons - the worker limit, the budget, + * nothing to choose - and those reach the headline. A refusal made by the + * container guard in DISPATCH did not: `queend` had said yes, so the report + * read "Started nothing. No reason given." under the headline "nothing to do", + * while the actual sentence sat one line lower where a headline reader never + * looks. + * + * TWO STRINGS, BECAUSE THEY GO TO TWO PLACES. The headline is served to a + * browser by `/queen/needs-you`, whose contract is no path, no branch and no + * connection detail - so it gets a label from a closed vocabulary and never + * the sentence, which names the workspace path. The body is the operator's, + * and it gets the guard's own SUMMARY: one sentence that carries every number + * and is written to fit (see `judgeBeeRoom`). Only outcomes the guard marked + * with `room` are read, so git output and provider bodies - which are also + * details of dispatches that did not start - can reach neither string. + * + * The summary is a FIELD and is never cut out of `detail`: the prose names a + * path, and a path may hold ". " - "/Volumes/Ext. Drive" ended the sentence + * after "Ext" while this function still split on it. + * + * It comes without a closing period because `nothingStartedLine` adds exactly + * one. One that is too long anyway is cut at a word and closed with two dots, + * which that period makes the ellipsis this file uses. + */ +export function containerRefusal( + outcomes: ReadonlyArray, +): { headline: string; sentence: string } | null { + const refused = outcomes.find( + (outcome) => outcome.started !== true && outcome.room !== undefined, + ) + if (!refused?.room) return null + const headline = `no room in the container: ${refused.room.resource}` + const summary = String(refused.room.summary ?? '') + .trim() + .replace(/[.!?\s]+$/, '') + const sentence = + summary.length <= DETAIL_SHOWN + ? summary + : `${summary.slice(0, DETAIL_SHOWN - 3).replace(/\s+\S*$/, '')}..` + return { headline, sentence: sentence || headline } +} + /** * The headline: escalations first, because a decision waiting on a human * outranks everything else; then the bees actually working, counted from the diff --git a/trios/agent-server/apps/server/src/api/services/queen-resources.ts b/trios/agent-server/apps/server/src/api/services/queen-resources.ts new file mode 100644 index 0000000000..2f0e3af4ca --- /dev/null +++ b/trios/agent-server/apps/server/src/api/services/queen-resources.ts @@ -0,0 +1,575 @@ +/** + * Whether the container can carry one more bee. + * + * THE MEASUREMENT, 2026-09-17. The container is limited to 24 GB. With sixteen + * bees running it held 14.7 GB, idle it holds one to three: about 0.85 GB per + * running bee. The same day the worker ceiling stopped being a constant and the + * credential list grew to thirty-four lanes. A full swarm would therefore have + * asked for about thirty GB of a twenty-four GB container, and the kernel + * answers that question by killing the container - every running bee with it, + * each holding the only copy of its unfinished turn. The volume is the second + * way to die: 50 GB, 31 in use, 36 at the day's peak. + * + * Nothing in dispatch looked at memory at all. The ceiling was a number, the + * credentials were a number, and the one thing that actually decides how many + * bees fit - the container - was not asked. + * + * So it is asked, before every start, and it is asked AGAIN every time: every + * variable and every reading below is taken on the call, never cached, so the + * line moves with the container and with the operator's variables and no + * restart is ever needed for the guard to notice either. + * + * WHAT THIS IS NOT. It is admission control. A bee that was admitted and then + * grows far past its reserve can still take the container down; the headroom is + * the only buffer against that. And it caps nothing by count - the policy and + * the credentials still decide how many bees MAY run; this only decides + * whether the next one FITS. + */ +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' + +export type MemoryReading = + | { + kind: 'measured' + usedBytes: number + limitBytes: number + /** Where the use was read: 'cgroup v2', 'cgroup v1' or '/proc/meminfo'. */ + source: string + /** Where the limit came from: 'cgroup', 'host MemTotal' or the variable. */ + limitSource: string + /** Said once in the log: a stated limit the kernel's own limit overrules. */ + note?: string + } + /** Not Linux: a development or test machine has nothing to read. */ + | { kind: 'unsupported'; platform: string } + /** Linux, and nothing usable. Unknown is not room. */ + | { kind: 'unknown'; tried: string[]; why?: string } + +export interface VolumeSpace { + totalBytes: number + freeBytes: number +} + +export interface ResourceSettings { + guardOn: boolean + beeMemoryBytes: number + memoryHeadroomPercent: number + warmupSeconds: number + beeDiskBytes: number + diskHeadroomPercent: number + memoryLimitBytes: number | null + /** One sentence per variable that was set to something unusable. */ + notes: string[] +} + +export type BeeRoom = + | { ok: true } + | { + ok: false + resource: 'memory' | 'disk' + /** + * The first sentence of `detail`, without its period: every number, no + * variable name, under 200 characters. A field rather than something a + * reader cuts out of the prose, because the prose names the workspace + * path and a path may contain ". " - "/Volumes/Ext. Drive" ended the + * sentence after "Ext" when the report split on it. + */ + summary: string + detail: string + } + +const MB = 1024 * 1024 +/** + * DECIMAL, for the headroom caps and for every number printed. Railway sells + * "24 GB" and its cgroup file says 24000000000; printed in binary units that is + * "22.4 GB", which reads as a wrong limit and invites an operator to "correct" + * it with TRIOS_QUEEN_MEMORY_LIMIT_MB. The `_MB` variables stay binary, as + * every memory flag an operator has met is. + */ +const GB = 1_000_000_000 +/** + * The headroom is a share of the limit, and a share of something huge is huge: + * ten percent of a developer's terabyte disk is a hundred GB "kept free", and + * the guard would refuse every local dispatch on a laptop with ninety GB to + * spare. The share protects a SMALL container; past these sizes it stops + * growing. + */ +const MEMORY_HEADROOM_CAP = 8 * GB +const DISK_HEADROOM_CAP = 10 * GB +/** cgroup v1 writes "no limit" as a number near 2^63. */ +const V1_UNLIMITED = 2 ** 60 + +function gb(bytes: number): string { + return (bytes / GB).toFixed(1) +} + +function readFileOrNull(path: string): string | null { + try { + return readFileSync(path, 'utf8') + } catch { + return null + } +} + +function statValue(stat: string | null, key: string): number { + if (!stat) return 0 + const match = new RegExp(`^${key}\\s+(\\d+)$`, 'm').exec(stat) + return match ? Number(match[1]) : 0 +} + +function meminfoBytes(meminfo: string | null, key: string): number | null { + if (!meminfo) return null + const match = new RegExp(`^${key}:\\s+(\\d+)\\s*kB`, 'm').exec(meminfo) + return match ? Number(match[1]) * 1024 : null +} + +/** + * Read on every call. A value that cannot be used is replaced by its default + * AND SAID SO: a variable that is silently ignored is the zero-length-key trap + * again - it looks configured in the editor and changes nothing. + */ +export function resourceSettings( + env: NodeJS.ProcessEnv = process.env, +): ResourceSettings { + const notes: string[] = [] + const bounded = ( + name: string, + fallback: number, + low: number, + high: number, + ): number => { + const raw = env[name]?.trim() + if (!raw) return fallback + const parsed = Number(raw) + if (!Number.isInteger(parsed) || parsed < low || parsed > high) { + notes.push( + `${name}=${raw.slice(0, 24)} is not an integer from ${low} to ${high}; using ${fallback}`, + ) + return fallback + } + return parsed + } + const guardRaw = env.TRIOS_QUEEN_RESOURCE_GUARD?.trim().toLowerCase() + if (guardRaw && guardRaw !== 'on' && guardRaw !== 'off') { + notes.push( + `TRIOS_QUEEN_RESOURCE_GUARD=${guardRaw.slice(0, 24)} is neither on nor off; the guard stays on`, + ) + } + // Not `bounded`: this one has no default to fall back TO. Unset, empty and 0 + // all mean "read the limit from the container", and an unusable value must + // say THAT - "using 0" would tell an operator the container has no memory. + const limitRaw = env.TRIOS_QUEEN_MEMORY_LIMIT_MB?.trim() + let limitMb = 0 + if (limitRaw && limitRaw !== '0') { + const parsed = Number(limitRaw) + if (Number.isInteger(parsed) && parsed >= 512 && parsed <= 4_194_304) { + limitMb = parsed + } else { + notes.push( + `TRIOS_QUEEN_MEMORY_LIMIT_MB=${limitRaw.slice(0, 24)} is not an integer from 512 to 4194304; ` + + 'using the limit the container reports', + ) + } + } + return { + guardOn: guardRaw !== 'off', + beeMemoryBytes: + bounded('TRIOS_QUEEN_BEE_MEMORY_MB', 1024, 128, 65_536) * MB, + memoryHeadroomPercent: bounded( + 'TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT', + 10, + 0, + 50, + ), + warmupSeconds: bounded('TRIOS_QUEEN_BEE_WARMUP_SECONDS', 600, 0, 3600), + beeDiskBytes: bounded('TRIOS_QUEEN_BEE_DISK_MB', 2048, 256, 1_048_576) * MB, + diskHeadroomPercent: bounded( + 'TRIOS_QUEEN_DISK_HEADROOM_PERCENT', + 10, + 0, + 50, + ), + memoryLimitBytes: limitMb > 0 ? limitMb * MB : null, + notes, + } +} + +/** + * What the container holds that the kernel CANNOT give back, against its limit. + * + * Not `memory.current`: that counts page cache, and a server that clones + * repositories and links node_modules fills the cache constantly. And not the + * kubelet's working set either (`current - inactive_file`), which was the first + * version: it still counts `active_file` and reclaimable slab, and those come + * back only under pressure AT the limit - which this guard refuses before ever + * reaching. On a classic-LRU kernel a page read twice is promoted to active and + * nothing demotes it without pressure, so the figure only grows: a review put + * 16 GB of active cache in a container with no bee running and the guard + * answered "19.6 GB in use" every round, for good, because a memory refusal + * frees nothing. What kills a container is memory that reclaim cannot free, so + * that is what is counted: current minus the file cache, both lists, minus + * reclaimable slab. `/proc/meminfo`'s MemAvailable already means the same. + * Mapped and dirty file pages it under-counts are what the headroom is for. + * + * The limit is the cgroup's. When the cgroup says "max" the container is not + * the thing that is limited and the host's MemTotal is the real ceiling; when + * even that is not what the platform enforces, TRIOS_QUEEN_MEMORY_LIMIT_MB + * states it, and replaces the LIMIT only - use is always measured. + */ +export function readContainerMemory( + readers: { + readFile?: (path: string) => string | null + platform?: string + env?: NodeJS.ProcessEnv + } = {}, +): MemoryReading { + const platform = readers.platform ?? process.platform + if (platform !== 'linux') return { kind: 'unsupported', platform } + const read = readers.readFile ?? readFileOrNull + const stated = resourceSettings(readers.env ?? process.env).memoryLimitBytes + const meminfo = read('/proc/meminfo') + const hostTotal = meminfoBytes(meminfo, 'MemTotal') + const tried: string[] = [] + + const hostUsed = + hostTotal !== null && meminfoBytes(meminfo, 'MemAvailable') !== null + ? hostTotal - (meminfoBytes(meminfo, 'MemAvailable') ?? 0) + : null + + const finish = ( + usedRaw: number, + cgroupLimit: number | null, + source: string, + ): MemoryReading | null => { + // THE KERNEL'S LIMIT WINS. A stated limit can only LOWER the line: the + // kernel kills at its own number whatever a variable says, so a stated + // 24576 MB over a cgroup limit of 24000000000 bytes would have moved the + // guard's line to within one bee of the kill line, and said nothing. + const ceiling = cgroupLimit ?? hostTotal + const limit = + stated && ceiling ? Math.min(stated, ceiling) : (stated ?? ceiling) + if (!limit || limit <= 0) return null + const statedWins = Boolean(stated) && limit === stated + // The cgroup says "max": the container is not what is limited, the machine + // is - and the machine is also used by everything else on it. Judging this + // cgroup's own use against the host's total admitted thirteen more bees + // into a host with half a GB available. + const onHost = cgroupLimit === null && !statedWins && hostUsed !== null + const used = onHost ? Math.max(usedRaw, hostUsed ?? 0) : usedRaw + return { + kind: 'measured', + usedBytes: Math.max(0, used), + limitBytes: limit, + source: onHost && used !== usedRaw ? '/proc/meminfo' : source, + limitSource: statedWins + ? 'TRIOS_QUEEN_MEMORY_LIMIT_MB' + : cgroupLimit + ? 'cgroup' + : 'host MemTotal', + ...(stated && !statedWins + ? { + note: + `TRIOS_QUEEN_MEMORY_LIMIT_MB states ${gb(stated)} GB, above the ` + + `${gb(limit)} GB the ${cgroupLimit ? 'kernel enforces' : 'machine has'}; ` + + 'using the smaller', + } + : {}), + } + } + + tried.push('cgroup v2') + const v2Current = read('/sys/fs/cgroup/memory.current') + if (v2Current !== null && /^\d+\s*$/.test(v2Current)) { + const max = (read('/sys/fs/cgroup/memory.max') ?? '').trim() + const stat = read('/sys/fs/cgroup/memory.stat') + const givenBack = + statValue(stat, 'inactive_file') + + statValue(stat, 'active_file') + + statValue(stat, 'slab_reclaimable') + const reading = finish( + Number(v2Current) - givenBack, + /^\d+$/.test(max) ? Number(max) : null, + 'cgroup v2', + ) + if (reading) return reading + } + + tried.push('cgroup v1') + const v1Usage = read('/sys/fs/cgroup/memory/memory.usage_in_bytes') + if (v1Usage !== null && /^\d+\s*$/.test(v1Usage)) { + const limitRaw = Number( + (read('/sys/fs/cgroup/memory/memory.limit_in_bytes') ?? '').trim(), + ) + const stat = read('/sys/fs/cgroup/memory/memory.stat') + const givenBack = + statValue(stat, 'total_inactive_file') + + statValue(stat, 'total_active_file') + const reading = finish( + Number(v1Usage) - givenBack, + Number.isFinite(limitRaw) && limitRaw > 0 && limitRaw < V1_UNLIMITED + ? limitRaw + : null, + 'cgroup v1', + ) + if (reading) return reading + } + + // MEASURED from inside a Railway container, 2026-09-17: memory.max reads + // 24000000000 and /proc/meminfo reads MemTotal 338 GB - the HOST. So this last + // resort describes the whole machine, which is the right thing to judge on a + // machine with no container around the server and the wrong thing to hold + // against a stated CONTAINER limit: the host's use is always far above it and + // every start would be refused for memory this container never touched. + tried.push('/proc/meminfo') + const available = meminfoBytes(meminfo, 'MemAvailable') + if (hostTotal !== null && available !== null) { + if (stated) { + return { + kind: 'unknown', + tried, + why: + 'TRIOS_QUEEN_MEMORY_LIMIT_MB states a container limit, but only ' + + '/proc/meminfo is readable and it measures the whole host', + } + } + return { + kind: 'measured', + usedBytes: Math.max(0, hostTotal - available), + limitBytes: hostTotal, + source: '/proc/meminfo', + limitSource: 'host MemTotal', + } + } + return { kind: 'unknown', tried } +} + +/** One line saying what the guard reads, for the log - not for a refusal. */ +export function describeReading(memory: MemoryReading): string { + if (memory.kind === 'unsupported') { + return `memory is not measured on ${memory.platform}; only the volume is` + } + if (memory.kind === 'unknown') { + return `memory cannot be measured (tried ${memory.tried.join(', ')})` + } + return ( + `memory from ${memory.source}, limit ${gb(memory.limitBytes)} GB ` + + `from ${memory.limitSource}` + ) +} + +/** + * Free bytes on the volume, from `df -Pk` - the same subprocess, for the same + * reason, as `volumeUsedPercent`: statfs arithmetic is exact on the Linux + * container and wrong on an APFS shared container, and a guard that refuses + * every dispatch in a unit test is a guard somebody switches off. + */ +export function volumeSpace( + dir: string, + run: (dir: string) => string = (target) => + execFileSync('df', ['-Pk', target], { + encoding: 'utf8', + timeout: 10_000, + stdio: ['ignore', 'pipe', 'ignore'], + }), +): VolumeSpace | null { + try { + const line = run(dir).trim().split('\n').pop() ?? '' + // Filesystem 1024-blocks Used Available Capacity Mounted-on + const match = /^\S+\s+(\d+)\s+\d+\s+(\d+)\s+\d+%/.exec(line) + if (!match) return null + const totalBytes = Number(match[1]) * 1024 + const freeBytes = Number(match[2]) * 1024 + return totalBytes > 0 ? { totalBytes, freeBytes } : null + } catch { + return null + } +} + +/** + * Bees that have started and not yet grown. + * + * The tick starts bees back to back, and a bee takes minutes to allocate what + * it will hold. Thirty-four starts in one round would each see the same nearly + * empty container and each be admitted; the reading is true and the conclusion + * is wrong. So every start inside the warm-up window reserves a bee's memory + * that the reading cannot show yet. + * + * Process-local on purpose: after a restart every bee has been reaped, so an + * empty list is the correct list. It reserves memory; it caps nothing by count. + */ +const youngBees = new Map() +/** No warm-up this module accepts is longer; past it an entry is only garbage. */ +const LONGEST_WARMUP_MS = 3600 * 1000 + +/** + * Keyed by the turn, because a reservation must end with the bee. `turn.ok` + * only means /chat answered 200, and a provider refusal arrives exactly that + * way: 200, then the failure in the stream. Kept as bare timestamps, ten such + * dead starts in ten minutes reserved ten GB for bees that no longer existed + * and the guard refused a nearly empty container, saying they were "still + * growing". `closeDispatch`, where every ending lands, calls `noteBeeEnded`. + */ +export function noteBeeStarted(turn: string, nowMs: number): void { + youngBees.set(turn, nowMs) +} + +export function noteBeeEnded(turn: string): void { + youngBees.delete(turn) +} + +export function youngBeeCount(nowMs: number, warmupSeconds: number): number { + const horizon = nowMs - warmupSeconds * 1000 + let young = 0 + for (const [turn, startedAt] of youngBees) { + // Dropped only past the LONGEST window, never past the caller's: the window + // is read on every call and may grow, and a count must not destroy what a + // later count would have needed. + if (nowMs - startedAt > LONGEST_WARMUP_MS) youngBees.delete(turn) + else if (startedAt > horizon) young += 1 + } + return warmupSeconds > 0 ? young : 0 +} + +export function resetYoungBees(): void { + youngBees.clear() +} + +/** + * One more bee: does it fit? + * + * Pure. Memory is judged first because it is the failure that takes everything + * with it, and ONE cause is reported - a sentence naming two is a sentence an + * operator acts on half of. Unknown is a refusal: a guard that reads an + * unmeasurable container as empty disables itself exactly when the container + * is unwell. Not-Linux passes, because there is no container to protect. + * + * The FIRST SENTENCE carries every number, names no variable and stays under + * 200 characters, because it is the part the round report quotes as the reason + * nothing started (`containerRefusal` in queen-report-lines.ts). What to do + * about it comes after. + */ +export function judgeBeeRoom(input: { + memory: MemoryReading + volume: VolumeSpace | null + youngBees: number + settings: ResourceSettings + volumeDir: string + reaped?: { removed: number; keptDirty: number; keptRunning: number } + /** + * The workspace directory does not exist: there is no checkout to fill, and + * `df` on a missing path fails like a sick filesystem does. The entrypoint + * starts the server even when the clone failed, and "no room in the + * container: disk" with nineteen GB free sent the operator to the wrong + * place while hiding the git error that names the real one. So disk is not + * judged, and `prepareWorktree` says what is actually wrong. + */ + checkoutMissing?: boolean +}): BeeRoom { + const { memory, volume, youngBees, settings, volumeDir, reaped } = input + + const refuse = ( + resource: 'memory' | 'disk', + summary: string, + advice: string, + ): BeeRoom => ({ + ok: false, + resource, + summary, + detail: `${summary}. ${advice}`, + }) + + if (memory.kind === 'unknown') { + return refuse( + 'memory', + `container memory cannot be measured (tried ${memory.tried.join(', ')}) ` + + 'and unknown is not room', + `${memory.why ? `${memory.why}. ` : ''}Not starting a bee blind; ` + + 'TRIOS_QUEEN_RESOURCE_GUARD=off dispatches without the guard.', + ) + } + if (memory.kind === 'measured') { + const keepFree = Math.min( + (memory.limitBytes * settings.memoryHeadroomPercent) / 100, + MEMORY_HEADROOM_CAP, + ) + const asked = (youngBees + 1) * settings.beeMemoryBytes + if (memory.usedBytes + asked > memory.limitBytes - keepFree) { + // Short on purpose: the first sentence is what a report line shows. + const where = + memory.limitSource === 'cgroup' + ? memory.source + : memory.limitSource === 'host MemTotal' + ? `${memory.source}, host total` + : `${memory.source}, stated limit` + // The window as it was set: "0 min" for twenty seconds told an operator + // the count was taken over nothing. + const window = + settings.warmupSeconds % 60 === 0 + ? `${settings.warmupSeconds / 60} min` + : `${settings.warmupSeconds} s` + const growing = + youngBees > 0 + ? ` and ${youngBees} bee(s) started in the last ${window} are still growing` + : '' + return refuse( + 'memory', + `container memory is ${gb(memory.usedBytes)} GB of ` + + `${gb(memory.limitBytes)} GB in use (${where})${growing}; another bee ` + + `needs about ${gb(settings.beeMemoryBytes)} GB and ${gb(keepFree)} GB ` + + 'stays free', + 'Not starting a bee that would get every running bee killed. Asked ' + + 'again every round, and a finishing bee is what gives memory back; ' + + 'TRIOS_QUEEN_BEE_MEMORY_MB and TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT ' + + '(MiB and percent) move the line.', + ) + } + } + + if (input.checkoutMissing) return { ok: true } + if (volume === null) { + return refuse( + 'disk', + `free space on ${volumeDir} cannot be measured (df gave nothing) and ` + + 'unknown is not room', + 'Not starting a bee blind; TRIOS_QUEEN_RESOURCE_GUARD=off dispatches ' + + 'without the guard.', + ) + } + const keepFree = diskKeptFree(volume, settings) + if (volume.freeBytes < settings.beeDiskBytes + keepFree) { + const reaping = reaped + ? `after a reap (${reaped.removed} removed; kept ${reaped.keptDirty} with ` + + `uncommitted work and ${reaped.keptRunning} of running bees)` + : '(nothing reaped yet)' + return refuse( + 'disk', + `volume ${volumeDir} has ${gb(volume.freeBytes)} GB free of ` + + `${gb(volume.totalBytes)} GB ${reaping}; another bee needs about ` + + `${gb(settings.beeDiskBytes)} GB and ${gb(keepFree)} GB stays free`, + 'Not starting a bee that would fill the disk under the running ones. ' + + 'TRIOS_QUEEN_BEE_DISK_MB and TRIOS_QUEEN_DISK_HEADROOM_PERCENT move the line.', + ) + } + return { ok: true } +} + +function diskKeptFree(volume: VolumeSpace, settings: ResourceSettings): number { + return Math.min( + (volume.totalBytes * settings.diskHeadroomPercent) / 100, + DISK_HEADROOM_CAP, + ) +} + +/** + * The used-percent at which the disk rule is satisfied, for the reaper to stop + * at. Without it the one reap the guard asks for ran to QUEEN_VOLUME_LOW - 55% + * - and removed every clean tree it was allowed to, when the guard needed seven + * GB. Rounded down, so stopping there is never stopping short. + */ +export function diskLineUsedPercent( + volume: VolumeSpace, + settings: ResourceSettings, +): number { + const need = settings.beeDiskBytes + diskKeptFree(volume, settings) + return Math.max(0, Math.floor(100 * (1 - need / volume.totalBytes))) +} diff --git a/trios/agent-server/apps/server/src/api/services/queen-tick.ts b/trios/agent-server/apps/server/src/api/services/queen-tick.ts index dc6cb55f85..65ad80f59e 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-tick.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-tick.ts @@ -60,6 +60,7 @@ import { releaseQueenLease, } from './queen-lease' import { + containerRefusal, type DispatchReportOutcome, dispatchesThatStarted, nothingStartedLine, @@ -1488,6 +1489,7 @@ export async function runRound( detail: string conversationId?: string keyIndex?: number + room?: { resource: 'memory' | 'disk'; summary: string } }> = [] let board = [...registry.rows[0].tasks, ...containerTasks] let current: QueendChoice | null = choice @@ -2589,18 +2591,29 @@ async function report( '.', ) } + const noRoom = containerRefusal(started) if (dispatchesThatStarted(started).length === 0) { // The refusal, verbatim. A round that started nothing is the case where a // summary in my own words would be the least trustworthy thing on the page. // This fires for a round that dispatched and was refused as it does for a // round that never dispatched - the sentence is the same because the fact - // (no bee started) is the same. The refused dispatches above carry the why. - lines.push(nothingStartedLine(choice.refusal, candidates)) + // (no bee started) is the same. When `queend` allowed the round and the + // CONTAINER refused it, the why is the guard's sentence, and it goes here + // as well as in the line above: "No reason given" next to a reason is a + // report contradicting itself. + lines.push( + nothingStartedLine(choice.refusal ?? noRoom?.sentence, candidates), + ) const skipped = (choice.skipped ?? []).slice(0, 6) if (skipped.length > 0) lines.push('', ...skipped.map((s) => ` ${s}`)) } - const headline = reportHeadline(escalated.length, started, choice.refusal) + // The label, never the sentence: this string is served to a browser. + const headline = reportHeadline( + escalated.length, + started, + choice.refusal ?? noRoom?.headline, + ) await pool .query( diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index a6e0a466c6..20cbae43c1 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -1,5 +1,18 @@ -import { afterEach, beforeAll, describe, expect, it } from 'bun:test' -import { mkdirSync, mkdtempSync, realpathSync, writeFileSync } from 'node:fs' +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'bun:test' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { Pool } from 'pg' @@ -18,12 +31,14 @@ import { missingProviderRefusal, POOL_KEY_STRIDE, prepareWorktree, + reapWorktrees, recordDispatch, resolveWorkerProvider, setDurableCloseListener, workerCapacityBreakdown, workspaceRoot, } from '../../src/api/services/queen-dispatch' +import { resetYoungBees } from '../../src/api/services/queen-resources' import { logger } from '../../src/lib/logger' const GENERIC_WORKER_KEYS = [ @@ -67,6 +82,16 @@ const KEYS = [ 'TRIOS_QUEEN_WORKER_CONTEXT', 'TRIOS_QUEEN_WORKER_LANES_PER_KEY', 'TRIOS_QUEEN_MAX_WORKERS', + // Every variable the container guard reads. One of them left in a developer's + // shell moves the line the fixtures below are written against, and a fixture + // that stops being "full" walks past the gate into a real `prepareWorktree`. + 'TRIOS_QUEEN_RESOURCE_GUARD', + 'TRIOS_QUEEN_BEE_MEMORY_MB', + 'TRIOS_QUEEN_MEMORY_HEADROOM_PERCENT', + 'TRIOS_QUEEN_BEE_WARMUP_SECONDS', + 'TRIOS_QUEEN_BEE_DISK_MB', + 'TRIOS_QUEEN_DISK_HEADROOM_PERCENT', + 'TRIOS_QUEEN_MEMORY_LIMIT_MB', // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. 'TRIOS_QUEEN_WORKER_API_KEY_17', 'TRIOS_QUEEN_WORKER_API_KEY_1024', @@ -744,6 +769,464 @@ describe('queen dispatch precheck', () => { }) }) +/** + * A key is free and a provider answers - and the container may still have no + * room. Measured on 2026-09-17: 24 GB, about 0.85 GB a bee, thirty-four lanes + * connected. Nothing in dispatch asked the container before this. + */ +describe('the container is asked before a worktree is cut', () => { + // Decimal, like every number the guard prints. + const GB = 1_000_000_000 + const full = () => + ({ + kind: 'measured', + usedBytes: 21.2 * GB, + limitBytes: 24 * GB, + source: 'cgroup v2', + limitSource: 'cgroup', + }) as const + const calm = () => ({ ...full(), usedBytes: 3 * GB }) + const roomy = () => ({ totalBytes: 50 * GB, freeBytes: 19 * GB }) + const oneFreeKey = () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'https://api.z.ai/api/paas/v4' + process.env.TRIOS_QUEEN_WORKER_API_KEY = 'a' + } + // An EMPTY directory stands where the workspace would be, for every case. A + // fixture that stops reading as "full" - a variable in somebody's shell moved + // the line - walks past the gate into the real `prepareWorktree`, and on the + // container the default workspace is the LIVE checkout: a unit test would + // fetch there and cut a worktree in it. + let previousWorkspace: string | undefined + beforeEach(() => { + previousWorkspace = process.env.WORKSPACE_DIR + process.env.WORKSPACE_DIR = realpathSync( + mkdtempSync(join(tmpdir(), 'queen-guard-')), + ) + }) + afterEach(() => { + if (previousWorkspace === undefined) delete process.env.WORKSPACE_DIR + else process.env.WORKSPACE_DIR = previousWorkspace + // Module state, shared by every test file in the bun process. + resetYoungBees() + }) + + const recordingPool = (running: number[] = []) => { + const queries: Array<{ text: string; values?: unknown[] }> = [] + const pool = { + query: async (text: string, values?: unknown[]) => { + queries.push({ text, values }) + return /finished_at IS NULL/.test(text) + ? { + rowCount: running.length, + rows: running.map((issue) => ({ issue })), + } + : { rowCount: 1, rows: [] } + }, + } as unknown as Pool + return { pool, queries } + } + + it('refuses on memory with the numbers, cuts nothing, and books nothing against the issue', async () => { + oneFreeKey() + const { pool, queries } = recordingPool() + let reaped = 0 + + const outcome = await dispatchBee( + pool, + 1500, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: full, + volume: roomy, + reap: async () => { + reaped += 1 + return { + before: 0, + after: 0, + removed: [], + keptDirty: [], + keptRunning: [], + refused: [], + } + }, + }, + ) + + expect(outcome.started).toBe(false) + expect(outcome.detail).toMatch( + /^container memory is 21\.2 GB of 24\.0 GB in use/, + ) + // Tagged with the guard's own sentence, because the report may quote this + // refusal and no other. + expect(outcome.room?.resource).toBe('memory') + expect(outcome.detail.startsWith(`${outcome.room?.summary}. `)).toBe(true) + // Memory cannot be freed by reaping, so the reaper is not even asked. + expect(reaped).toBe(0) + // It says something about the CONTAINER, and once the swarm is memory-bound + // it is how nearly every round ends. A row per round would archive history + // and wipe the issue's last attempt for a reason that is not the issue's. + expect(queries.filter((q) => /queen_dispatch/.test(q.text))).toEqual([]) + }) + + it('gives the disk one reap that protects running bees and its own tree, then names what was kept', async () => { + oneFreeKey() + const { pool } = recordingPool([7, 12]) + const calls: Array<{ + protect?: ReadonlySet + high?: number + low?: number + }> = [] + + const outcome = await dispatchBee( + pool, + 1501, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => ({ totalBytes: 50 * GB, freeBytes: 5.1 * GB }), + reap: async (opts = {}) => { + calls.push(opts) + return { + before: 90, + after: 90, + removed: ['/w/.worktrees/queen-3'], + keptDirty: [], + keptRunning: ['/w/.worktrees/queen-7', '/w/.worktrees/queen-12'], + refused: [], + } + }, + }, + ) + + expect(calls).toHaveLength(1) + // queen-1501 is the issue being dispatched. It is not running, so without + // its own name here a re-dispatch could reap the tree it was about to reuse + // and then cut it afresh with -B, dropping the last attempt's commits. + expect([...(calls[0].protect ?? [])].sort()).toEqual([ + 'queen-12', + 'queen-1501', + 'queen-7', + ]) + // It starts whatever the percentage (high 0) and stops where the rule is + // met - 2048 MiB + 5 GB of 50 GB is 85% used - not at the collector's 55%. + expect(calls[0].high).toBe(0) + expect(calls[0].low).toBe(85) + expect(outcome.started).toBe(false) + expect(outcome.room?.resource).toBe('disk') + expect(outcome.detail).toContain( + 'has 5.1 GB free of 50.0 GB after a reap (1 removed; kept 0 with uncommitted work and 2 of running bees)', + ) + }) + + it('reaps nothing when it cannot find out who is running', async () => { + // An unreadable registry used to mean "protect nobody": with twenty bees + // running, that reap takes the clean trees out from under most of them. + oneFreeKey() + const pool = { + query: async (text: string) => { + if (/finished_at IS NULL/.test(text)) { + throw new Error('Connection terminated unexpectedly') + } + return { rowCount: 1, rows: [] } + }, + } as unknown as Pool + let reaps = 0 + const outcome = await dispatchBee( + pool, + 1505, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => ({ totalBytes: 50 * GB, freeBytes: 5.1 * GB }), + reap: async () => { + reaps += 1 + throw new Error('the reaper must not be called') + }, + }, + ) + expect(reaps).toBe(0) + expect(outcome.started).toBe(false) + expect(outcome.room?.resource).toBe('disk') + expect(outcome.detail).toContain('(nothing reaped yet)') + }) + + it('starts the bee when the one reap made the room', async () => { + // The half of the disk path that ADMITS. With it broken - the second + // judgement reusing the first reading, or refusing whatever it reads - a + // volume past the line can never recover: the gate sits in front of the only + // reaper there was, so every round would refuse on disk for good. + oneFreeKey() + const { pool } = recordingPool() + let reaps = 0 + const outcome = await dispatchBee( + pool, + 1504, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => + reaps === 0 ? { totalBytes: 50 * GB, freeBytes: 5.1 * GB } : roomy(), + reap: async () => { + reaps += 1 + return { + before: 90, + after: 60, + removed: ['/w/.worktrees/queen-3'], + keptDirty: [], + keptRunning: [], + refused: [], + } + }, + }, + ) + expect(reaps).toBe(1) + // Past the gate: what stops it next is the empty workspace, not the guard. + expect(outcome.room).toBeUndefined() + expect(outcome.detail).not.toContain('GB free of') + }) + + /** A repository to cut worktrees from, standing where the workspace is. */ + const hive = (): { root: string; git: (args: string[]) => string } => { + const root = join(process.env.WORKSPACE_DIR ?? '', 'BrowserOS') + const git = (args: string[], cwd = root) => { + const done = Bun.spawnSync(['git', ...args], { + cwd, + env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' }, + }) + if (done.exitCode !== 0) { + throw new Error( + `git ${args.join(' ')} failed: ${done.stderr.toString()}`, + ) + } + return done.stdout.toString().trim() + } + mkdirSync(root) + git(['init', '-q', '-b', 'dev']) + git(['config', 'user.email', 'bee@example.com']) + git(['config', 'user.name', 'Bee']) + writeFileSync(join(root, 'README.md'), 'hive\n') + git(['add', '.']) + git(['-c', 'commit.gpgsign=false', 'commit', '-qm', 'first']) + git(['remote', 'add', 'origin', root]) + git(['fetch', '-q', 'origin']) + return { root, git } + } + /** Everything a real start reads from the environment, put back afterwards. */ + const realStarts = () => { + const names = [ + 'TRIOS_API_TOKEN', + 'TRIOS_REPO_URL', + 'TRIOS_REPO_REF', + 'QUEEN_VOLUME_KEEP', + ] + const previous = names.map((name) => process.env[name]) + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + delete process.env.TRIOS_REPO_URL + delete process.env.TRIOS_REPO_REF + const realFetch = globalThis.fetch + // /chat answers 200 and the stream STAYS OPEN, as a working bee's does: a + // stream that is already over ends the bee, and an ended bee reserves + // nothing. + const streams: Array> = [] + globalThis.fetch = (async () => + new Response( + new ReadableStream({ + start(controller) { + streams.push(controller) + }, + }), + )) as unknown as typeof fetch + const finish = async (index: number) => { + streams[index].enqueue( + new TextEncoder().encode('data: {"type":"finish"}\n\n'), + ) + streams[index].close() + delete streams[index] + // The drain closes the dispatch on its own turn of the loop. + await new Promise((resolve) => setTimeout(resolve, 50)) + } + return { + finish, + restore: async () => { + globalThis.fetch = realFetch + for (const index of Object.keys(streams)) await finish(Number(index)) + names.forEach((name, i) => { + if (previous[i] === undefined) delete process.env[name] + else process.env[name] = previous[i] + }) + }, + } + } + + it('reserves for the bee it just started, and gives it back the moment that bee ends', async () => { + // The wiring, not the arithmetic: a start that is not noted, or a judgement + // that does not count the young, admits thirty-four bees back to back on + // one true reading of a nearly empty container - the incident this exists + // to prevent. So this drives REAL starts: a repository to cut the worktree + // from, and `/chat` answering 200. + oneFreeKey() + hive() + const turns = realStarts() + const { pool } = recordingPool() + // 19.7 of 24 GB: one more bee fits under the line (21.6), two do not. + const nearlyFull = () => ({ ...full(), usedBytes: 19.7 * GB }) + const start = 1_789_000_000_000 + const go = (issue: number, ms: number) => + dispatchBee(pool, issue, 'brief', [], [], undefined, [], 'none', { + memory: nearlyFull, + volume: roomy, + now: () => ms, + // Not this machine's disk: at 95% full the cut itself is refused. + volumeUsed: () => 40, + }) + try { + const first = await go(1510, start) + expect(first.detail).toContain('zai/') + expect(first.started).toBe(true) + + const second = await go(1511, start + 60_000) + expect(second.started).toBe(false) + expect(second.room?.resource).toBe('memory') + expect(second.room?.summary).toContain( + '1 bee(s) started in the last 10 min are still growing', + ) + + // The first bee ends - a provider refusal ends one in seconds - and its + // reservation ends with it, two minutes in and not ten. + await turns.finish(0) + const third = await go(1512, start + 120_000) + expect(third.started).toBe(true) + + // The third is still running and still young: no room for another... + expect((await go(1513, start + 180_000)).started).toBe(false) + // ...until it has had its ten minutes, and its memory is in the reading. + expect((await go(1513, start + 13 * 60_000)).started).toBe(true) + } finally { + await turns.restore() + } + }) + + it('never reaps the tree of the issue it is about to dispatch again', async () => { + // A redeploy killed the swarm; #1520 was reaped at boot and its tree holds + // one committed, unpushed attempt. The container carries no push + // credential, so that commit is the only copy. With the volume short the + // gate reaps - and #1520 is by definition not running. Reaped, its tree + // would be cut afresh with -B and the commit would drop off the branch. + oneFreeKey() + const { root, git } = hive() + const turns = realStarts() + process.env.QUEEN_VOLUME_KEEP = '0' + for (const branch of ['queen-1520', 'queen-3']) { + git([ + 'worktree', + 'add', + '-q', + '-b', + branch, + join(root, '.worktrees', branch), + 'dev', + ]) + } + const tree = join(root, '.worktrees', 'queen-1520') + writeFileSync(join(tree, 'attempt.md'), 'the only copy\n') + Bun.spawnSync(['git', 'add', '.'], { cwd: tree }) + Bun.spawnSync( + [ + 'git', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'the first attempt', + ], + { cwd: tree, env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' } }, + ) + const attempt = git(['rev-parse', 'queen-1520']) + const { pool } = recordingPool() + let reaps = 0 + try { + const outcome = await dispatchBee( + pool, + 1520, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: calm, + volume: () => + reaps === 0 + ? { totalBytes: 50 * GB, freeBytes: 5.1 * GB } + : roomy(), + // The real reaper against the real repository, on a fixed reading. + reap: async (opts = {}) => { + reaps += 1 + return reapWorktrees({ ...opts, volumeUsed: () => 90 }) + }, + volumeUsed: () => 40, + }, + ) + expect(reaps).toBe(1) + // The reap did run, and did remove what nobody needs... + expect(existsSync(join(root, '.worktrees', 'queen-3'))).toBe(false) + // ...and the issue's own tree was reused with its commit where it was. + expect(outcome.detail).toMatch(/^reused an existing worktree/) + expect(git(['rev-parse', 'queen-1520'])).toBe(attempt) + expect(outcome.started).toBe(true) + } finally { + await turns.restore() + } + }) + + it('can be switched off, and then a full container refuses nothing here', async () => { + oneFreeKey() + process.env.TRIOS_QUEEN_RESOURCE_GUARD = 'off' + const { pool } = recordingPool() + const outcome = await dispatchBee( + pool, + 1503, + 'brief', + [], + [], + undefined, + [], + 'none', + { + memory: full, + volume: () => null, + }, + ) + // Whatever stops it next (there is no repository in this directory), it is + // not the guard. + expect(outcome.room).toBeUndefined() + expect(outcome.detail).not.toContain('container memory is') + expect(outcome.detail).not.toContain('unknown is not room') + }) +}) + /** * #1308. `workers.capacity` answers a number; this breakdown answers what the * number is MADE of. An operator seeing capacity 4 cannot act on it without @@ -1121,6 +1604,58 @@ describe('draining a turn', () => { ).toBe(true) }) + it('gives the agent back when the turn is over, so the memory comes with it', async () => { + // Measured on the deployed container 2026-09-18: seven bees cost 2.5 GB + // between them, and an hour later, with none running, it held 11 GB and + // never gave it back. Every dispatch builds a session; nothing asked for + // one back, so the baseline grew with every bee that FINISHED. + const { pool } = recordingPool() + const asked: Array<{ url: string; method?: string; auth?: string }> = [] + const realFetch = globalThis.fetch + const previous = process.env.TRIOS_API_TOKEN + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + globalThis.fetch = (async ( + url: string | URL | Request, + init?: RequestInit, + ) => { + asked.push({ + url: String(url), + method: init?.method, + auth: new Headers(init?.headers).get('Authorization') ?? undefined, + }) + return new Response('{}', { status: 200 }) + }) as unknown as typeof fetch + try { + await drain(pool, sse([{ type: 'finish' }]), 'the-turn-that-ended', 1244) + } finally { + globalThis.fetch = realFetch + if (previous === undefined) delete process.env.TRIOS_API_TOKEN + else process.env.TRIOS_API_TOKEN = previous + } + expect(asked).toHaveLength(1) + expect(asked[0].method).toBe('DELETE') + expect(asked[0].url).toEndWith('/chat/the-turn-that-ended') + expect(asked[0].auth).toBe('Bearer a-token-for-the-test') + }) + + it('still closes the dispatch when the session will not be released', async () => { + const { pool, asked } = recordingPool() + const realFetch = globalThis.fetch + const previous = process.env.TRIOS_API_TOKEN + process.env.TRIOS_API_TOKEN = 'a-token-for-the-test' + globalThis.fetch = (async () => { + throw new Error('connection refused') + }) as unknown as typeof fetch + try { + await drain(pool, sse([{ type: 'finish' }]), 'conv-9', 1244) + } finally { + globalThis.fetch = realFetch + if (previous === undefined) delete process.env.TRIOS_API_TOKEN + else process.env.TRIOS_API_TOKEN = previous + } + expect(touching(asked, 'UPDATE queen_dispatch')).toHaveLength(1) + }) + it('reports no price rather than a free turn when the frame never came', async () => { const { pool, asked } = recordingPool() await drain(pool, sse([{ type: 'text-delta', delta: 'killed' }]), 'c', 1244) diff --git a/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts b/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts index f23b1c7766..da31b8b602 100644 --- a/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-report-lines.test.ts @@ -6,18 +6,25 @@ * The #1379 regression test: a round whose dispatches were all refused must * report that nothing started, not report a bee working. * - * Imports `bun:test` and the module under test and nothing else. The module - * has no imports by design - a bee's worktree has no `node_modules`, so this - * test has to run with neither a database nor the `queend` binary to hand. + * Imports `bun:test`, the module under test and one neighbour that itself + * imports only Node builtins. The module has no imports by design - a bee's + * worktree has no `node_modules`, so this test has to run with neither a + * database nor the `queend` binary to hand. */ import { describe, expect, it } from 'bun:test' import { + containerRefusal, dispatchesThatStarted, nothingStartedLine, refusedLines, reportHeadline, startedLine, } from '../../src/api/services/queen-report-lines' +// Builtins only, like the module above: it loads in a bare worktree too. +import { + judgeBeeRoom, + resourceSettings, +} from '../../src/api/services/queen-resources' /** The refusal `dispatchBee` returns when every provider key is busy - the * routine case from the issue, not an exotic failure. */ @@ -131,3 +138,127 @@ describe('queen report dispatch lines', () => { expect(short).toBe('Refused #8: no.') }) }) + +describe('a round queend allowed and the container refused', () => { + const GB = 1_000_000_000 + const settings = resourceSettings({}) + /** What the guard REALLY returns, at the sizes that make it longest: a + * hand-shortened sample is how a cut in the middle of a number went unseen. */ + const refusal = (room: ReturnType, issue = 1500) => { + if (room.ok) throw new Error('the fixture was meant to have no room') + return { + started: false, + issue, + detail: room.detail, + room: { resource: room.resource, summary: room.summary }, + } + } + const memoryFull = () => + judgeBeeRoom({ + memory: { + kind: 'measured', + usedBytes: 1234.5 * GB, + limitBytes: 4096 * GB, + source: '/proc/meminfo', + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 1000, + settings: { ...settings, beeMemoryBytes: 64 * GB, warmupSeconds: 3599 }, + volumeDir: '/workspace/BrowserOS', + }) + const diskShort = ( + volumeDir = '/workspace/BrowserOS', + reaped = { removed: 1234, keptDirty: 567, keptRunning: 1024 }, + ) => + judgeBeeRoom({ + memory: { kind: 'unsupported', platform: 'darwin' }, + volume: { totalBytes: 50 * GB, freeBytes: 5.1 * GB }, + youngBees: 0, + settings, + volumeDir, + reaped, + }) + + it('quotes the whole summary in the line that read "No reason given"', () => { + for (const room of [memoryFull(), diskShort()]) { + const said = containerRefusal([refusal(room)]) + // Whole: it ends where the guard's sentence ends, not mid-word. + expect(said?.sentence.endsWith('GB stays free')).toBe(true) + expect(said?.sentence.length).toBeLessThanOrEqual(200) + const line = nothingStartedLine(said?.sentence, 4) + expect(line).not.toContain('No reason given') + expect(line).toContain('GB stays free. 4 issue(s) were on the table.') + } + }) + + it('is not fooled by a path that holds a period and a space', () => { + // The sentence used to be cut out of the prose at the first ". ", and the + // prose names the workspace: the report read "volume /Volumes/Ext." + const said = containerRefusal([ + refusal( + diskShort('/Volumes/Ext. Drive/ws/BrowserOS', { + removed: 2, + keptDirty: 1, + keptRunning: 14, + }), + ), + ]) + expect(said?.sentence).toContain( + '/Volumes/Ext. Drive/ws/BrowserOS has 5.1 GB', + ) + expect(said?.sentence.endsWith('GB stays free')).toBe(true) + }) + + it('gives the headline a label and never the sentence', () => { + const outcomes = [refusal(diskShort())] + const headline = reportHeadline( + 0, + outcomes, + containerRefusal(outcomes)?.headline, + ) + expect(headline).toBe('no room in the container: disk') + // /queen/needs-you serves this string to a browser: no path, no numbers. + expect(headline).not.toContain('/workspace') + }) + + it('reads no other refusal, because those details carry git and provider output', () => { + expect( + containerRefusal([ + { + started: false, + issue: 7, + detail: + "git worktree add failed: fatal: '/workspace/BrowserOS/.worktrees/queen-7' already exists", + }, + { started: false, issue: 8, detail: KEY_EXHAUSTED_DETAIL }, + ]), + ).toBeNull() + expect( + containerRefusal([{ started: true, issue: 1, detail: 'fresh worktree' }]), + ).toBeNull() + expect(containerRefusal([])).toBeNull() + }) + + it('closes a summary that is too long anyway with the ellipsis this file uses', () => { + const said = containerRefusal([ + { + started: false, + issue: 9, + detail: 'irrelevant', + room: { + resource: 'disk', + summary: `volume /${'deep/'.repeat(60)} has 1.0 GB free`, + }, + }, + ]) + expect(said?.sentence.length).toBeLessThanOrEqual(200) + expect(nothingStartedLine(said?.sentence, 1)).toContain('... 1 issue(s)') + // And a refusal that somehow carries no summary still says something true. + expect( + containerRefusal([ + { started: false, issue: 1, room: { resource: 'memory' } }, + ])?.sentence, + ).toBe('no room in the container: memory') + }) +}) diff --git a/trios/agent-server/apps/server/tests/api/queen-resources.test.ts b/trios/agent-server/apps/server/tests/api/queen-resources.test.ts new file mode 100644 index 0000000000..429a9c7af6 --- /dev/null +++ b/trios/agent-server/apps/server/tests/api/queen-resources.test.ts @@ -0,0 +1,595 @@ +/** + * @license + * Copyright 2025 BrowserOS + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * WHY THIS EXISTS. + * + * 2026-09-17: the container is limited to 24 GB and held 14.7 GB with sixteen + * bees running - about 0.85 GB a bee. The same day the worker ceiling stopped + * being a constant and the credential list reached thirty-four lanes. A full + * swarm would have asked for about thirty GB, and the kernel answers that by + * killing the container with every running bee in it. Nothing in dispatch + * looked at memory at all. + * + * These pin the judge with the container in states a test machine never is in: + * every reading is injected, so none of this needs Linux, a cgroup or a full + * disk to be true. + */ +import { afterEach, describe, expect, it } from 'bun:test' +import { + describeReading, + diskLineUsedPercent, + judgeBeeRoom, + type MemoryReading, + noteBeeEnded, + noteBeeStarted, + readContainerMemory, + resetYoungBees, + resourceSettings, + volumeSpace, + youngBeeCount, +} from '../../src/api/services/queen-resources' + +// DECIMAL, like every number the module prints and the platform sells: the +// Railway container is 24000000000 bytes and everyone calls it 24 GB. +const GB = 1_000_000_000 +const files = (map: Record) => (path: string) => + path in map ? map[path] : null +const measured = (usedGb: number, limitGb = 24): MemoryReading => ({ + kind: 'measured', + usedBytes: usedGb * GB, + limitBytes: limitGb * GB, + source: 'cgroup v2', + limitSource: 'cgroup', +}) +const roomy = { totalBytes: 50 * GB, freeBytes: 19 * GB } +const defaults = resourceSettings({}) + +afterEach(() => resetYoungBees()) + +describe('reading the container', () => { + it('counts what the kernel cannot give back, not the cache it will', () => { + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '16000000000\n', + '/sys/fs/cgroup/memory.max': '25769803776\n', + '/sys/fs/cgroup/memory.stat': + 'anon 9\ninactive_file 2000000000\nactive_file 5\n', + }), + }) + expect(reading).toEqual({ + kind: 'measured', + usedBytes: 16_000_000_000 - 2_000_000_000 - 5, + limitBytes: 25_769_803_776, + source: 'cgroup v2', + limitSource: 'cgroup', + }) + }) + + it('does not refuse an idle container for a day of page cache', () => { + // No bee running: 2.5 GB of heap, and 16 GB of ACTIVE cache plus 2.5 GB of + // reclaimable slab left by a day of clones and installs. Counted as the + // kubelet counts (current - inactive_file) that is 21 GB "in use", past the + // line, and a memory refusal frees nothing - so every round refused until a + // redeploy. + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '23000000000', + '/sys/fs/cgroup/memory.max': '24000000000', + '/sys/fs/cgroup/memory.stat': + 'anon 2500000000\nfile 18000000000\ninactive_file 2000000000\n' + + 'active_file 16000000000\nslab_reclaimable 2500000000\nslab_unreclaimable 100\n', + }), + }) + expect(reading).toMatchObject({ usedBytes: 2_500_000_000 }) + expect( + judgeBeeRoom({ + memory: reading, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }), + ).toEqual({ ok: true }) + }) + + it('takes the host as the ceiling when the cgroup says max, and the variable when stated', () => { + const readFile = files({ + '/sys/fs/cgroup/memory.current': '3000000000', + '/sys/fs/cgroup/memory.max': 'max', + '/proc/meminfo': + 'MemTotal: 67108864 kB\nMemAvailable: 60000000 kB\n', + }) + const host = readContainerMemory({ platform: 'linux', env: {}, readFile }) + expect(host).toMatchObject({ + limitBytes: 67_108_864 * 1024, + limitSource: 'host MemTotal', + }) + // The variable replaces the LIMIT only. Use is still measured. + const stated = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile, + }) + expect(stated).toMatchObject({ + usedBytes: 3_000_000_000, + limitBytes: 24_576 * 1024 * 1024, + source: 'cgroup v2', + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }) + }) + + it('reads cgroup v1, and treats its near-2^63 limit as no limit', () => { + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory/memory.usage_in_bytes': '5000000000', + '/sys/fs/cgroup/memory/memory.limit_in_bytes': '9223372036854771712', + '/sys/fs/cgroup/memory/memory.stat': 'total_inactive_file 1000000000\n', + '/proc/meminfo': 'MemTotal: 33554432 kB\nMemAvailable: 31457280 kB\n', + }), + }) + expect(reading).toMatchObject({ + usedBytes: 4_000_000_000, + limitBytes: 33_554_432 * 1024, + source: 'cgroup v1', + limitSource: 'host MemTotal', + }) + }) + + it('judges the MACHINE when the cgroup says max, because everything else on it uses it too', () => { + // An unlimited container on a shared 16 GB machine: this cgroup holds one + // GB, the rest of the machine holds fifteen. Its own use against the + // host's total admitted thirteen more bees into half a GB. + const reading = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '1000000000', + '/sys/fs/cgroup/memory.max': 'max', + '/proc/meminfo': 'MemTotal: 16000000 kB\nMemAvailable: 500000 kB\n', + }), + }) + expect(reading).toMatchObject({ + usedBytes: 15_500_000 * 1024, + limitBytes: 16_000_000 * 1024, + source: '/proc/meminfo', + limitSource: 'host MemTotal', + }) + }) + + it('lets a stated limit lower the line and never raise it past what the kernel enforces', () => { + const railway = files({ + '/sys/fs/cgroup/memory.current': '23500000000', + '/sys/fs/cgroup/memory.max': '24000000000', + '/proc/meminfo': 'MemTotal: 338368112 kB\nMemAvailable: 109626516 kB\n', + }) + // 24576 MiB is 25.8 GB: the natural way to write "24 GB", and above the + // kernel's 24000000000. Taken at its word it admitted a bee at 23.5 GB. + const above = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile: railway, + }) + expect(above).toMatchObject({ + limitBytes: 24_000_000_000, + limitSource: 'cgroup', + }) + if (above.kind !== 'measured') throw new Error('expected a reading') + expect(above.note).toContain( + 'states 25.8 GB, above the 24.0 GB the kernel enforces', + ) + expect( + judgeBeeRoom({ + memory: above, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }).ok, + ).toBe(false) + // Below it, the variable is the line and says so. + const below = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '16384' }, + readFile: railway, + }) + expect(below).toMatchObject({ + limitBytes: 16_384 * 1024 * 1024, + limitSource: 'TRIOS_QUEEN_MEMORY_LIMIT_MB', + }) + if (below.kind === 'measured') expect(below.note).toBeUndefined() + }) + + it('falls back to /proc/meminfo, floors use at zero, and says unknown when nothing reads', () => { + expect( + readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/proc/meminfo': 'MemTotal: 1000 kB\nMemAvailable: 400 kB\n', + }), + }), + ).toMatchObject({ usedBytes: 600 * 1024, source: '/proc/meminfo' }) + expect( + readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '1024' }, + readFile: files({ + '/sys/fs/cgroup/memory.current': '100', + '/sys/fs/cgroup/memory.stat': 'inactive_file 500\n', + }), + }), + ).toMatchObject({ usedBytes: 0 }) + expect( + readContainerMemory({ platform: 'linux', env: {}, readFile: () => null }), + ).toEqual({ + kind: 'unknown', + tried: ['cgroup v2', 'cgroup v1', '/proc/meminfo'], + }) + }) + + it('will not hold the HOST against a stated container limit', () => { + // Measured inside a Railway container on 2026-09-17: memory.max reads + // 24000000000 while /proc/meminfo reads the machine - MemTotal 338 GB, 228 + // of them in use. Judged against a stated 24 GB that is "full" forever. + const railwayHost = + 'MemTotal: 338368112 kB\nMemAvailable: 109626516 kB\n' + const reading = readContainerMemory({ + platform: 'linux', + env: { TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }, + readFile: files({ '/proc/meminfo': railwayHost }), + }) + expect(reading.kind).toBe('unknown') + const refusal = judgeBeeRoom({ + memory: reading, + volume: { totalBytes: 50 * GB, freeBytes: 19 * GB }, + youngBees: 0, + settings: resourceSettings({}), + volumeDir: '/workspace', + }) + expect(refusal.ok).toBe(false) + if (!refusal.ok) { + expect(refusal.detail).toContain('it measures the whole host') + expect(refusal.detail).toContain('TRIOS_QUEEN_RESOURCE_GUARD=off') + } + // With nothing stated the same files ARE the machine, and that is judged. + expect( + readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ '/proc/meminfo': railwayHost }), + }), + ).toMatchObject({ kind: 'measured', limitSource: 'host MemTotal' }) + // And what Railway really shows is read as the container it is. + const container = readContainerMemory({ + platform: 'linux', + env: {}, + readFile: files({ + '/sys/fs/cgroup/memory.current': '26374144', + '/sys/fs/cgroup/memory.max': '24000000000', + '/sys/fs/cgroup/memory.stat': 'inactive_file 2363392\n', + '/proc/meminfo': railwayHost, + }), + }) + expect(container).toMatchObject({ + usedBytes: 26_374_144 - 2_363_392, + limitBytes: 24_000_000_000, + limitSource: 'cgroup', + }) + expect(describeReading(container)).toBe( + 'memory from cgroup v2, limit 24.0 GB from cgroup', + ) + }) + + it('has nothing to read on a machine that is not Linux, and says which', () => { + expect(readContainerMemory({ platform: 'darwin' })).toEqual({ + kind: 'unsupported', + platform: 'darwin', + }) + }) + + it('parses df -Pk, and returns null - not zero - when it cannot', () => { + const line = + 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/x 52428800 32505856 19922944 62% /workspace\n' + expect(volumeSpace('/workspace', () => line)).toEqual({ + totalBytes: 52_428_800 * 1024, + freeBytes: 19_922_944 * 1024, + }) + expect(volumeSpace('/workspace', () => 'garbage')).toBeNull() + expect( + volumeSpace('/workspace', () => { + throw new Error('df is not there') + }), + ).toBeNull() + expect(volumeSpace(process.cwd())?.freeBytes).toBeGreaterThan(0) + expect(volumeSpace('/definitely/not/a/path/on/this/machine')).toBeNull() + }) +}) + +describe('the variables', () => { + it('have defaults that protect a deployment which sets none of them', () => { + expect(defaults).toMatchObject({ + guardOn: true, + beeMemoryBytes: 1024 ** 3, + memoryHeadroomPercent: 10, + warmupSeconds: 600, + beeDiskBytes: 2 * 1024 ** 3, + diskHeadroomPercent: 10, + memoryLimitBytes: null, + notes: [], + }) + }) + + it('replace an unusable value by the default and SAY SO', () => { + for (const bad of ['abc', '0', '-5', '999999999', '1.5']) { + const settings = resourceSettings({ TRIOS_QUEEN_BEE_MEMORY_MB: bad }) + expect(settings.beeMemoryBytes).toBe(1024 ** 3) + expect(settings.notes.join(' ')).toContain('TRIOS_QUEEN_BEE_MEMORY_MB') + } + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'maybe' }), + ).toMatchObject({ guardOn: true }) + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'maybe' }).notes, + ).toHaveLength(1) + expect( + resourceSettings({ TRIOS_QUEEN_RESOURCE_GUARD: 'OFF' }).guardOn, + ).toBe(false) + }) + + it('never tell an operator the container limit is 0', () => { + for (const bad of ['24GB', '256', '-1', '1.5']) { + const settings = resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: bad }) + expect(settings.memoryLimitBytes).toBeNull() + expect(settings.notes).toHaveLength(1) + expect(settings.notes[0]).toContain( + 'using the limit the container reports', + ) + expect(settings.notes[0]).not.toContain('using 0') + } + // Unset, empty and 0 are three spellings of "not stated": no note at all. + for (const unset of [undefined, '', ' ', '0']) { + const settings = resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: unset }) + expect(settings.memoryLimitBytes).toBeNull() + expect(settings.notes).toHaveLength(0) + } + expect( + resourceSettings({ TRIOS_QUEEN_MEMORY_LIMIT_MB: '24576' }) + .memoryLimitBytes, + ).toBe(24_576 * 1024 * 1024) + }) + + it('are read on every call, so moving the line needs no restart', () => { + const env: NodeJS.ProcessEnv = {} + expect(resourceSettings(env).beeMemoryBytes).toBe(1024 ** 3) + env.TRIOS_QUEEN_BEE_MEMORY_MB = '2048' + expect(resourceSettings(env).beeMemoryBytes).toBe(2 * 1024 ** 3) + }) +}) + +describe('does one more bee fit', () => { + const judge = (memory: MemoryReading, youngBees = 0, volume = roomy) => + judgeBeeRoom({ + memory, + volume, + youngBees, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + }) + + it('admits a bee at the measured sixteen-bee load and refuses near the limit', () => { + expect(judge(measured(14.7))).toEqual({ ok: true }) + const refused = judge(measured(20.9)) + expect(refused.ok).toBe(false) + if (refused.ok) return + expect(refused.resource).toBe('memory') + expect(refused.detail).toMatch( + /^container memory is 20\.9 GB of 24\.0 GB in use \(cgroup v2\);/, + ) + expect(refused.detail).toContain('needs about 1.1 GB and 2.4 GB stays free') + expect(refused.detail.endsWith('.')).toBe(true) + // The report quotes `summary`: it is the first sentence of the detail, it + // carries every number, it names no variable and it fits a report line. + expect(refused.detail.startsWith(`${refused.summary}. `)).toBe(true) + expect(refused.summary).toContain('20.9 GB of 24.0 GB') + expect(refused.summary).toContain('1.1 GB and 2.4 GB stays free') + expect(refused.summary).not.toContain('TRIOS_') + expect(refused.summary.length).toBeLessThanOrEqual(200) + }) + + it('reserves for bees that started and have not grown, and says how many', () => { + const refused = judge(measured(10), 12) + expect(refused.ok).toBe(false) + if (refused.ok) return + expect(refused.detail).toContain( + '12 bee(s) started in the last 10 min are still growing', + ) + expect( + judge(measured(20.9)).ok === false && !judge(measured(20.9)).ok, + ).toBe(true) + const calm = judge(measured(20.9)) + if (!calm.ok) expect(calm.detail).not.toContain('still growing') + }) + + it('stops a round of back-to-back starts that all read the same empty container', () => { + // The reading does not move: a bee takes minutes to allocate what it will + // hold. Thirty-four starts would each be admitted on the reading alone. + let started = 0 + const nowMs = 1_000_000 + while (started < 34) { + const verdict = judgeBeeRoom({ + memory: measured(1), + volume: roomy, + youngBees: youngBeeCount(nowMs, defaults.warmupSeconds), + settings: defaults, + volumeDir: '/workspace', + }) + if (!verdict.ok) break + noteBeeStarted(`turn-${started}`, nowMs) + started += 1 + } + // (24 - 2.4 - 1) GB over 1024 MiB a bee. + expect(started).toBe(19) + }) + + it('forgets a start once the bee has had time to grow', () => { + noteBeeStarted('a', 0) + noteBeeStarted('b', 1_000) + noteBeeStarted('c', 700_000) + expect(youngBeeCount(701_000, 600)).toBe(1) + expect(youngBeeCount(701_000, 0)).toBe(0) + // A count never destroys what a later count needs: the window is read on + // every call and may have grown since. + expect(youngBeeCount(701_000, 3600)).toBe(3) + }) + + it('frees the reservation when the bee ends, however young it was', () => { + // `turn.ok` only means /chat answered 200; a provider refusal arrives as a + // 200 whose stream then fails. Ten such dead starts in ten minutes used to + // reserve ten GB for bees that no longer existed. + for (let i = 0; i < 18; i++) noteBeeStarted(`dead-${i}`, 1_000) + expect(judge(measured(1.9), youngBeeCount(2_000, 600)).ok).toBe(false) + for (let i = 0; i < 18; i++) noteBeeEnded(`dead-${i}`) + expect(youngBeeCount(2_000, 600)).toBe(0) + expect(judge(measured(1.9), youngBeeCount(2_000, 600))).toEqual({ + ok: true, + }) + // Ending a turn nobody noted is not an error. + noteBeeEnded('never-started') + }) + + it('names the window as it was set, not rounded to minutes', () => { + const at = (warmupSeconds: number) => { + const refused = judgeBeeRoom({ + memory: measured(10), + volume: roomy, + youngBees: 12, + settings: { ...defaults, warmupSeconds }, + volumeDir: '/workspace', + }) + return refused.ok ? '' : refused.summary + } + expect(at(20)).toContain('started in the last 20 s are') + expect(at(90)).toContain('started in the last 90 s are') + expect(at(600)).toContain('started in the last 10 min are') + }) + + it("refuses on disk with the reaper's counts in the sentence", () => { + const short = { totalBytes: 50 * GB, freeBytes: 5.1 * GB } + const first = judge(measured(3), 0, short) + expect(first.ok).toBe(false) + if (first.ok) return + expect(first.resource).toBe('disk') + expect(first.detail).toContain( + 'has 5.1 GB free of 50.0 GB (nothing reaped yet)', + ) + const after = judgeBeeRoom({ + memory: measured(3), + volume: short, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + reaped: { removed: 2, keptDirty: 1, keptRunning: 14 }, + }) + if (after.ok) throw new Error('expected a refusal') + expect(after.detail).toContain( + 'after a reap (2 removed; kept 1 with uncommitted work and 14 of running bees)', + ) + expect(after.detail).toContain('needs about 2.1 GB and 5.0 GB stays free') + // Where the reaper may stop: 2048 MiB + 5 GB of 50 GB is 85.7% used. + expect(diskLineUsedPercent(short, defaults)).toBe(85) + // A path may hold ". ", so the report reads the field and never the prose. + const dotted = judgeBeeRoom({ + memory: measured(3), + volume: short, + youngBees: 0, + settings: defaults, + volumeDir: '/Volumes/Ext. Drive/ws/BrowserOS', + }) + if (dotted.ok) throw new Error('expected a refusal') + expect(dotted.summary).toContain( + '/Volumes/Ext. Drive/ws/BrowserOS has 5.1 GB', + ) + expect(dotted.summary.endsWith('GB stays free')).toBe(true) + }) + + it('leaves a checkout that is not there to the code that can say so', () => { + // The entrypoint starts the server even when the clone failed. `df` on the + // missing directory gives nothing, and "no room in the container: disk" + // with nineteen GB free hid the git error that names the real cause. + expect( + judgeBeeRoom({ + memory: measured(3), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + checkoutMissing: true, + }), + ).toEqual({ ok: true }) + // Memory is still judged: a full container is full wherever the checkout is. + expect( + judgeBeeRoom({ + memory: measured(23), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace/BrowserOS', + checkoutMissing: true, + }).ok, + ).toBe(false) + }) + + it('does not let a share of a huge disk refuse a laptop with ninety GB to spare', () => { + const laptop = { totalBytes: 1000 * GB, freeBytes: 90 * GB } + expect( + judge({ kind: 'unsupported', platform: 'darwin' }, 0, laptop), + ).toEqual({ ok: true }) + }) + + it('treats unknown as a refusal and not-Linux as nothing to protect', () => { + const blind = judge({ + kind: 'unknown', + tried: ['cgroup v2', 'cgroup v1', '/proc/meminfo'], + }) + if (blind.ok) throw new Error('expected a refusal') + expect(blind.detail).toContain('unknown is not room') + expect(blind.detail).toContain('TRIOS_QUEEN_RESOURCE_GUARD=off') + const noDisk = judgeBeeRoom({ + memory: measured(3), + volume: null, + youngBees: 0, + settings: defaults, + volumeDir: '/workspace', + }) + if (noDisk.ok) throw new Error('expected a refusal') + expect(noDisk.resource).toBe('disk') + expect(judge({ kind: 'unsupported', platform: 'darwin' })).toEqual({ + ok: true, + }) + }) + + it('reports one cause, memory first, and leaks nothing of the closed breakdown', () => { + const both = judge(measured(23.5), 0, { + totalBytes: 50 * GB, + freeBytes: 1 * GB, + }) + if (both.ok) throw new Error('expected a refusal') + expect(both.resource).toBe('memory') + for (const word of [ + 'connectedCredentials', + 'lanesPerCredential', + 'effectiveCapacity', + 'API_KEY', + ]) { + expect(both.detail).not.toContain(word) + } + }) +}) diff --git a/trios/agent-server/apps/server/tests/api/queen-round.test.ts b/trios/agent-server/apps/server/tests/api/queen-round.test.ts index 62134ff6cd..11c8be6410 100644 --- a/trios/agent-server/apps/server/tests/api/queen-round.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-round.test.ts @@ -1,6 +1,12 @@ import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { spawnSync } from 'node:child_process' -import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import type { Pool } from 'pg' @@ -137,9 +143,20 @@ function roundPool(finished: FinishedRow[] = []) { const saved: Record = {} const realFetch = globalThis.fetch +/** What the one case that connects a credential sets, so no later case inherits it. */ +const WORKER_POOL_KEYS = [ + 'TRIOS_QUEEN_WORKER_PROVIDER', + 'TRIOS_QUEEN_WORKER_BASE_URL', + 'TRIOS_QUEEN_WORKER_API_KEY', + 'TRIOS_QUEEN_RESOURCE_GUARD', + 'TRIOS_QUEEN_BEE_MEMORY_MB', + 'TRIOS_QUEEN_BEE_DISK_MB', +] + beforeEach(() => { for (const key of [ ...PROVIDER_KEYS, + ...WORKER_POOL_KEYS, QUEEND_ENV, 'WORKSPACE_DIR', 'TRIOS_GITHUB_REPO', @@ -257,6 +274,54 @@ describe('queen round, lease lost', () => { ) }) +/** + * A round the policy ALLOWED and the container refused. + * + * `queend` says yes, a credential is free, and the container has no room. The + * report used to read "Started nothing. No reason given." under the headline + * "nothing to do" - with the guard's sentence one line lower, in the "Refused" + * line, where a headline reader never looks. The pure helpers are pinned in + * queen-report-lines.test.ts; this pins that the round actually calls them, which + * a review showed no test did: both call sites could be reverted to + * `choice.refusal` with every suite green. + */ +describe('queen round, no room in the container', () => { + it.if(present)( + 'says so in the report, with a label in the headline and the sentence in the body', + async () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'https://api.z.ai/api/paas/v4' + process.env.TRIOS_QUEEN_WORKER_API_KEY = 'a' + // No test machine has room for a 64 GiB bee holding a 1 TiB worktree. On + // Linux the refusal is memory; where memory is not measured it is disk - + // measured on a workspace that EXISTS, because a missing one is left for + // `prepareWorktree` to report. Either way the container refuses. + process.env.TRIOS_QUEEN_BEE_MEMORY_MB = '65536' + process.env.TRIOS_QUEEN_BEE_DISK_MB = '1048576' + process.env.WORKSPACE_DIR = realpathSync( + mkdtempSync(join(tmpdir(), 'queen-round-no-room-')), + ) + mkdirSync(join(process.env.WORKSPACE_DIR, 'BrowserOS')) + const { pool, queries } = roundPool() + await runRound(pool, 'me', 7, { held: true }, [ISSUE]) + + // Nothing is booked against the issue: the refusal is about the + // container, and it is how nearly every memory-bound round ends. + expect(queries.some((q) => isDispatchInsert(q.sql))).toBe(false) + const report = queries.find((q) => + q.sql.includes('INSERT INTO queen_report'), + ) + expect(report).toBeDefined() + const [headline, body] = report?.params as [string, string] + expect(headline).toMatch(/^no room in the container: (memory|disk)$/) + expect(body).not.toContain('No reason given') + expect(body).toMatch( + /Started nothing\. (container memory|free space on|volume) .+\. \d+ issue\(s\) were on the table\./, + ) + }, + ) +}) + /** * The send-back count, end to end, through the real policy. * diff --git a/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts b/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts index 89c27a95bf..6c62a38d38 100644 --- a/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-volume-gc.test.ts @@ -1,5 +1,18 @@ import { describe, expect, it } from 'bun:test' -import { volumeUsedPercent } from '../../src/api/services/queen-dispatch' +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + prepareWorktree, + reapWorktrees, + volumeUsedPercent, +} from '../../src/api/services/queen-dispatch' // WHY THIS EXISTS. // @@ -12,10 +25,12 @@ import { volumeUsedPercent } from '../../src/api/services/queen-dispatch' // while the application is unhealthy - which it was, BECAUSE the volume was // full. The collector depended on the thing whose failure it collects for. // -// These pin the properties that make the in-container collector safe. The -// removal path itself needs a real filesystem and a real git, so it is exercised -// in production behind the watermark rather than mocked into a shape that would -// agree with whatever it was written against. +// These pin the properties that make the in-container collector safe. Most of +// them read the source, because the removal path needs a real filesystem and a +// real git. The one property a reading could not pin - WHOSE tree is removed - +// runs against a real scratch repository at the bottom of this file: a review +// deleted the `continue` after `keptRunning.push` and every text assertion +// stayed green while a running bee's tree was removed and reported as kept. describe('volumeUsedPercent', () => { it('measures a directory that exists', () => { @@ -29,13 +44,16 @@ describe('volumeUsedPercent', () => { // UNKNOWN IS NOT ROOM. A guard that reads an unmeasurable disk as empty // disables itself exactly when the filesystem is unwell, which is the one // moment it is needed. - expect(volumeUsedPercent('/definitely/not/a/path/on/this/machine')).toBeNull() + expect( + volumeUsedPercent('/definitely/not/a/path/on/this/machine'), + ).toBeNull() }) }) describe('the reaper refuses the things that would lose work', () => { const source = Bun.file( - new URL('../../src/api/services/queen-dispatch.ts', import.meta.url).pathname, + new URL('../../src/api/services/queen-dispatch.ts', import.meta.url) + .pathname, ) it('never removes a worktree holding uncommitted work', async () => { @@ -46,7 +64,21 @@ describe('the reaper refuses the things that would lose work', () => { const fn = text.slice(text.indexOf('export async function reapWorktrees')) const body = fn.slice(0, fn.indexOf('\n}\n')) expect(body).toContain('keptDirty') - expect(body).toMatch(/dirty\.code !== 0 \|\| dirty\.out\.trim\(\)\.length > 0/) + expect(body).toMatch( + /dirty\.code !== 0 \|\| dirty\.out\.trim\(\)\.length > 0/, + ) + }) + + it('asks whether a bee is running BEFORE it asks whether the tree is dirty', async () => { + // A running bee's tree is kept whatever its status reads, and the status + // call is a subprocess per tree that a protected tree should not cost. + const text = await source.text() + const fn = text.slice(text.indexOf('export async function reapWorktrees')) + const body = fn.slice(0, fn.indexOf('\n}\n')) + expect(body.indexOf('opts.protect?.has(')).toBeGreaterThan(-1) + expect(body.indexOf('opts.protect?.has(')).toBeLessThan( + body.indexOf("['status', '--porcelain']"), + ) }) it('treats an unreadable status as dirty, not as clean', async () => { @@ -81,12 +113,182 @@ describe('the dispatch refuses before it dies half-way', () => { // The dispatch that exposed this died with "cannot update the ref ... unable // to write file", which sent every reader looking at git rather than at df. const text = await Bun.file( - new URL('../../src/api/services/queen-dispatch.ts', import.meta.url).pathname, + new URL('../../src/api/services/queen-dispatch.ts', import.meta.url) + .pathname, ).text() - const prep = text.slice(text.indexOf('export async function prepareWorktree')) + const prep = text.slice( + text.indexOf('export async function prepareWorktree'), + ) const beforeFetch = prep.slice(0, prep.indexOf("['fetch'")) expect(beforeFetch).toContain('volumeUsedPercent') expect(beforeFetch).toContain('reapWorktrees') expect(beforeFetch).toMatch(/volume \$\{still\}% full after reaping/) }) }) + +/** + * A real repository with real worktrees, because "whose tree was removed" is a + * fact about a filesystem. `origin.git` is bare, `BrowserOS` is the Queen's + * checkout - the name `workspaceRoot()` derives - and every name given becomes + * a clean worktree under `.worktrees/`, which is where dispatch cuts them. + */ +function scratchRepository(branches: string[]): { + root: string + tree: (branch: string) => string + restore: () => void +} { + const REAL_GIT = Bun.which('git') || '/usr/bin/git' + const git = (cwd: string, args: string[]): void => { + const done = Bun.spawnSync([REAL_GIT, ...args], { + cwd, + env: { ...process.env, GIT_CONFIG_GLOBAL: '/dev/null' }, + }) + if (done.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${done.stderr.toString()}`) + } + } + const previous = { + WORKSPACE_DIR: process.env.WORKSPACE_DIR, + TRIOS_REPO_REF: process.env.TRIOS_REPO_REF, + TRIOS_REPO_URL: process.env.TRIOS_REPO_URL, + QUEEN_VOLUME_KEEP: process.env.QUEEN_VOLUME_KEEP, + // The marks the cases below are written against (80 and 55). Left as the + // shell has them, QUEEN_VOLUME_HIGH=90 makes "85% reaps" a lie. + QUEEN_VOLUME_HIGH: process.env.QUEEN_VOLUME_HIGH, + QUEEN_VOLUME_LOW: process.env.QUEEN_VOLUME_LOW, + } + const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'queen-reap-'))) + const origin = join(scratch, 'origin.git') + git(scratch, ['init', '-q', '--bare', 'origin.git']) + const seed = join(scratch, 'seed') + mkdirSync(seed) + git(seed, ['init', '-q', '-b', 'dev']) + git(seed, ['config', 'user.email', 'bee@example.com']) + git(seed, ['config', 'user.name', 'Bee']) + writeFileSync(join(seed, 'README.md'), 'one good branch\n') + git(seed, ['add', '.']) + git(seed, ['-c', 'commit.gpgsign=false', 'commit', '-qm', 'first']) + git(seed, ['push', '-q', origin, 'dev']) + git(origin, ['symbolic-ref', 'HEAD', 'refs/heads/dev']) + const root = join(scratch, 'BrowserOS') + git(scratch, ['clone', '-q', origin, 'BrowserOS']) + const tree = (branch: string) => join(root, '.worktrees', branch) + for (const branch of branches) { + git(root, [ + 'worktree', + 'add', + '-q', + '-b', + branch, + tree(branch), + 'origin/dev', + ]) + } + process.env.WORKSPACE_DIR = scratch + process.env.TRIOS_REPO_REF = 'origin/dev' + delete process.env.TRIOS_REPO_URL + // The default keeps the newest six whoever they belong to, which in a + // repository of three trees is all of them: nothing would be a candidate and + // the test would pass with the protection deleted. + process.env.QUEEN_VOLUME_KEEP = '0' + delete process.env.QUEEN_VOLUME_HIGH + delete process.env.QUEEN_VOLUME_LOW + return { + root, + tree, + restore: () => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }, + } +} + +describe('the reaper, on a real repository', () => { + it('keeps the clean tree of a running bee and removes the one nobody is running', async () => { + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + const gc = await reapWorktrees({ + high: 0, + low: 0, + volumeUsed: () => 90, + protect: new Set(['queen-7']), + }) + expect(gc.keptRunning).toEqual([repo.tree('queen-7')]) + expect(gc.removed).toEqual([repo.tree('queen-3')]) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(false) + } finally { + repo.restore() + } + }) + + it('protects them in the reap prepareWorktree runs, which fires before the container guard does', async () => { + // 85% used: above QUEEN_VOLUME_HIGH, and on a 50 GB volume still 7.5 GB + // free - room as far as the guard is concerned. This reap was the + // unprotected one: two running bees lost their trees to it in a scratch + // reproduction before `running` existed. + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + let asked = 0 + const prepared = await prepareWorktree(99, { + volumeUsed: () => 85, + running: async () => { + asked += 1 + return new Set(['queen-7']) + }, + }) + expect(prepared.ok).toBe(true) + expect(asked).toBe(1) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(false) + expect(existsSync(repo.tree('queen-99'))).toBe(true) + } finally { + repo.restore() + } + }) + + it('reaps nothing when it cannot find out who is running', async () => { + // Null is "the registry could not be read". It used to arrive as an empty + // set - "nobody is running" - and the reap went ahead unprotected. + const repo = scratchRepository(['queen-3', 'queen-7']) + try { + const prepared = await prepareWorktree(97, { + volumeUsed: () => 85, + running: async () => null, + }) + expect(prepared.ok).toBe(true) + expect(existsSync(repo.tree('queen-3'))).toBe(true) + expect(existsSync(repo.tree('queen-7'))).toBe(true) + // And a volume that is truly full still refuses, saying why nothing went. + const full = await prepareWorktree(96, { + volumeUsed: () => 97, + running: async () => null, + }) + expect(full.ok).toBe(false) + expect(full.detail).toContain('volume 97% full and nothing was reaped') + } finally { + repo.restore() + } + }) + + it('does not ask who is running when there is nothing to reap', async () => { + const repo = scratchRepository(['queen-3']) + try { + let asked = 0 + const prepared = await prepareWorktree(98, { + volumeUsed: () => 40, + running: async () => { + asked += 1 + return new Set() + }, + }) + expect(prepared.ok).toBe(true) + expect(asked).toBe(0) + expect(existsSync(repo.tree('queen-3'))).toBe(true) + } finally { + repo.restore() + } + }) +})