diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f23b2d2..acea709f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.237.0 + +**A refused spawn says which budget channels ran short and by how much.** A `budget-exhausted` reservation now carries `shortfalls`: every channel that did not fit, each as `{ channel, requested, free }` (`ReservationShortfall`, exported). `scope.spawn` passes them through, and `spawn_worker` returns them with a reason a driver can act on, for example `the run pool refused this spawn: iterations has 58 free (this spawn asked for budget.maxIterations 100); budget.maxIterations at most 58 fits`. + +Iterations can be requested at exactly `free`, because a driver's own turns charge none. Tokens and dollars cannot: the driver's next turn is metered from the same pool before its retry reaches admission, so the text says to ask for well under `free`. A channel closed by unmeasured spend (`closedByUnknownSpend`) says the run admits no further spawn at any budget. + +Measured 2026-09-16 on a Discovery director placed on the Tangle sandbox: its first research child asked for 100 iterations against a 60-iteration pool, got "the run has no allocation left to give this worker", spent a throwaway 3-iteration probe worker to learn the pool still had room, and moved its research to local processes. + +That same sentence was also the reply for every other refusal: `max-live-workers`, `depth-exceeded`, `duplicate-key`, `key-conflict`, `invalid-identity`, and `scope-aborted` were each reported as an empty pool. Each now names its own cause and a next step a driver can take. The `usd-unbudgeted`, `in-doubt`, and `scope-settled` texts are unchanged, and `usd-unbudgeted` still yields to an exhausted channel as before. + +Resource validation (a child missing a root resource, a unit mismatch, an undeclared resource) now throws before any shortfall is computed, where before a short resource could return first. A consumer that compared a refusal with `toEqual({ ok: false, reason: 'budget-exhausted' })` now also receives `shortfalls`. + ## 0.236.0 A root that ran to completion under budget, selected nothing, and **never spawned a child** now settles `no-winner` with reason `no-children-spawned`. It used to settle `all-children-down` with `downCount: 0`, which reads as a fleet failure to anyone who did not open the journal. Fifteen sandbox-placed directors settled that way in one week while the actual fault was that the root never recursed, and every reader went looking at the fleet. diff --git a/api-surface.json b/api-surface.json index f89c35d0..1dec3263 100644 --- a/api-surface.json +++ b/api-surface.json @@ -786,7 +786,7 @@ "BridgeSeam": "type 3ee6c139b2d1", "Budget": "type d3e20424e44d", "BudgetOverspend": "type 034717e51be1", - "BudgetPool": "type 423107bf3e84", + "BudgetPool": "type d4f7ca28e55f", "BudgetPoolRestore": "type 09960d15dfe8", "BudgetReadout": "type 0e2ea9923511", "BudgetReconcileFault": "value 9e6d36f63406", @@ -1127,6 +1127,7 @@ "ReproductionCheck": "type d866d16ccd22", "ReservationHolder": "type 589e95ab1fb1", "ReservationRejection": "type 95613aa46396", + "ReservationShortfall": "type 307614981aeb", "ReservationStage": "type f37f95f31e56", "ReservationTicket": "type 58a154a39d5e", "ResolveDriveHarness": "type 14e24449c8f2", @@ -1201,7 +1202,7 @@ "SandboxSteeringOptions": "type 526a58a22f51", "SandboxToolPartState": "type 26cdd7a50965", "SandboxUsageLedger": "type bb9c3e368071", - "Scope": "type edc3d5f40e17", + "Scope": "type 7d484633ce30", "ScopeAnalyst": "type 1cd5ae8d0d15", "ScopeAnalyzeInput": "type 98b6562915c6", "ScopeArgs": "type e739f4591d93", diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 70030c28..49a19289 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.236.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.237.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -423,7 +423,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 167 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 957 exports. +Import from `@tangle-network/agent-runtime/kernel` — 958 exports. | Symbol | Kind | Summary | |---|---|---| @@ -954,6 +954,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 957 exports. | `RegistryAnalyzeProjection` | interface | Project a `ScopeAnalyzeInput` into the `AnalystRegistry.run` arguments. The registry runs over a | | `RenderCorpusToInstructionsOptions` | interface | Project accreted corpus facts into an `AgentProfile`'s instruction seams — the learning-flywheel | | `ReservationHolder` | interface | Who holds a reservation. Recorded at `reserve` and refined through `attribute` once admission | +| `ReservationShortfall` | interface | One budget channel a `budget-exhausted` reservation could not fit, with the amounts that | | `ReservationTicket` | interface | Opaque, single-use reservation handle returned by `reserve` and consumed by | | `ResolvedMcpServerLaunch` | interface | The spawn-ready strings for one stdio MCP server: profile config values | | `ResolvedSupervisorProfile` | interface | The exact profile fields consumed by supervisor materialization. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index b657e0cd..7e0bc340 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -12674,6 +12674,38 @@ The spawned node's id, once admission minted one. Absent for a reservation that *** +### ReservationShortfall + +One budget channel a `budget-exhausted` reservation could not fit, with the amounts that +decided it. A refusal lists every channel that did not fit (`shortfalls`), so shrinking one +request is not answered by a second refusal on a channel the caller was never told about +(observed live: a director whose 100-iteration child was refused spent a probe worker to learn +the pool still admitted 3). `free` is a snapshot: live reservations return to it as their +workers settle, and a driver's own metered turns draw tokens and dollars from the same pool +before its next request arrives, so only `iterations` can be requested at exactly `free`. +`closedByUnknownSpend` means work with unmeasured usage ran under that enforced limit; the +channel then refuses every reservation for the rest of the run. + +#### Properties + +##### channel + +> `readonly` **channel**: `"tokens"` \| `"iterations"` \| `"usd"` \| `` `resource:${string}` `` + +##### requested + +> `readonly` **requested**: `number` + +##### free + +> `readonly` **free**: `number` + +##### closedByUnknownSpend? + +> `readonly` `optional` **closedByUnknownSpend?**: `true` + +*** + ### BudgetPoolRestore State recovered from a prior process before new work is admitted. `committed` is measured spend @@ -12699,7 +12731,7 @@ while the public readout remains explicitly unknown. ##### reserve() -> **reserve**(`b`, `holder?`): \{ `ok`: `true`; `ticket`: [`ReservationTicket`](#reservationticket); \} \| \{ `ok`: `false`; `reason`: [`ReservationRejection`](#reservationrejection); \} +> **reserve**(`b`, `holder?`): \{ `ok`: `true`; `ticket`: [`ReservationTicket`](#reservationticket); \} \| \{ `ok`: `false`; `reason`: [`ReservationRejection`](#reservationrejection); `shortfalls?`: readonly [`ReservationShortfall`](#reservationshortfall)[]; \} Atomically reserve a child's full ceiling from the free balance. Fails closed ({ ok: false }) when the pool can't cover standard or named channels — the @@ -12717,7 +12749,7 @@ caller inspects `ok` before `ticket`. ###### Returns -\{ `ok`: `true`; `ticket`: [`ReservationTicket`](#reservationticket); \} \| \{ `ok`: `false`; `reason`: [`ReservationRejection`](#reservationrejection); \} +\{ `ok`: `true`; `ticket`: [`ReservationTicket`](#reservationticket); \} \| \{ `ok`: `false`; `reason`: [`ReservationRejection`](#reservationrejection); `shortfalls?`: readonly [`ReservationShortfall`](#reservationshortfall)[]; \} ##### attribute() @@ -22031,7 +22063,7 @@ One tree-wide view of simultaneous spawned work. Every nested scope reads the sa ##### spawn() -> **spawn**\<`C`\>(`agent`, `task`, `opts`): \{ `ok`: `true`; `handle`: [`Handle`](#handle-3)\<`C`\>; `prior?`: [`SpawnPrior`](#spawnprior)\<`C`\>; \} \| \{ `ok`: `false`; `reason`: [`SpawnRejection`](#spawnrejection); \} +> **spawn**\<`C`\>(`agent`, `task`, `opts`): \{ `ok`: `true`; `handle`: [`Handle`](#handle-3)\<`C`\>; `prior?`: [`SpawnPrior`](#spawnprior)\<`C`\>; \} \| \{ `ok`: `false`; `reason`: [`SpawnRejection`](#spawnrejection); `shortfalls?`: readonly [`ReservationShortfall`](#reservationshortfall)[]; \} Spawn a child. For a fresh key or an unkeyed spawn, tree-wide worker admission happens before a lazy factory is called, so a full worker allocation creates no worker, executor, or reservation. @@ -22065,7 +22097,7 @@ work: it returns the committed result on `prior` (see `SpawnOpts.key`). ###### Returns -\{ `ok`: `true`; `handle`: [`Handle`](#handle-3)\<`C`\>; `prior?`: [`SpawnPrior`](#spawnprior)\<`C`\>; \} \| \{ `ok`: `false`; `reason`: [`SpawnRejection`](#spawnrejection); \} +\{ `ok`: `true`; `handle`: [`Handle`](#handle-3)\<`C`\>; `prior?`: [`SpawnPrior`](#spawnprior)\<`C`\>; \} \| \{ `ok`: `false`; `reason`: [`SpawnRejection`](#spawnrejection); `shortfalls?`: readonly [`ReservationShortfall`](#reservationshortfall)[]; \} ##### next() @@ -23044,7 +23076,7 @@ One channel on which a settled reservation's measured spend exceeded what it res ##### channel -> `readonly` **channel**: [`SpendChannel`](#spendchannel) \| `"iterations"` +> `readonly` **channel**: `"iterations"` \| [`SpendChannel`](#spendchannel) ##### reserved diff --git a/docs/canonical-api.md b/docs/canonical-api.md index cd92c9a2..541a0e00 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.236.0.** +> **Version 0.237.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.182.0 <0.183.0`. > `sandbox` must satisfy `>=0.36.4 <0.42.0`. diff --git a/package.json b/package.json index 18365d1e..8413137c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.236.0", + "version": "0.237.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index f06fbd5a..7885b497 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -27,11 +27,12 @@ import type { ResultBlobStore, Scope, Settled, + SpawnRejection, Spend, Agent as SuperviseAgent, WorkerTraceEvidence, } from '../../runtime' -import { assertValidBudget } from '../../runtime/supervise/budget' +import { assertValidBudget, type ReservationShortfall } from '../../runtime/supervise/budget' import type { DeliverableSpec } from '../../runtime/supervise/completion-gate' import { type WatchTraceOptions, watchTrace } from '../../runtime/supervise/detector-monitor' import { freeSlots } from '../../runtime/supervise/dispatch' @@ -1430,6 +1431,84 @@ function spawnProfileArg(): Record { return spawnProfileArgCache } +const BUDGET_FIELD: Readonly> = { + tokens: 'maxTokens', + iterations: 'maxIterations', + usd: 'maxUsd', +} + +function budgetField(channel: ReservationShortfall['channel']): string { + return channel.startsWith('resource:') + ? `resources.${channel.slice('resource:'.length)}.limit` + : BUDGET_FIELD[channel as 'tokens' | 'iterations' | 'usd'] +} + +/** One clause per short channel. Iterations can be requested at exactly `free`: a driver's own + * turns charge none. Tokens and dollars cannot: the driver's next turn is metered from the same + * pool before its retry reaches admission, so the clause says to leave room. */ +function shortfallClause(shortfall: ReservationShortfall): string { + const { channel, requested, free } = shortfall + const field = budgetField(channel) + if (shortfall.closedByUnknownSpend === true) { + return `${channel} is closed: work with unmeasured ${channel} usage ran under the run's enforced limit, so this run admits no further spawn at any budget` + } + const asked = `this spawn asked for budget.${field} ${requested}` + if (free === 0) { + return `${channel} has nothing free (${asked}); an unused reservation returns only when its live worker settles` + } + if (channel === 'iterations' || channel.startsWith('resource:')) { + return `${channel} has ${free} free (${asked}); budget.${field} at most ${free} fits` + } + return `${channel} has ${free} free right now (${asked}); your own turns draw ${channel} from this same pool before a retry is admitted, so ask for well under ${free}` +} + +/** + * The reason text `spawn_worker` returns for a refused spawn. Every rejection kind names its own + * cause: a live-worker cap, a depth limit, or a key collision is not an empty budget, and telling a + * driver "no allocation left" for those sends it after the wrong fix. A `budget-exhausted` refusal + * names every channel that did not fit and the amounts, so the driver can size its next request. + */ +export function spawnRefusalReason( + reason: SpawnRejection, + shortfalls: readonly ReservationShortfall[] | undefined, + pinned: { + readonly usdUnbudgeted: string + readonly inDoubt: string + readonly scopeSettled: string + }, +): string { + switch (reason) { + case 'usd-unbudgeted': + return pinned.usdUnbudgeted + case 'in-doubt': + return pinned.inDoubt + case 'scope-settled': + return pinned.scopeSettled + case 'scope-aborted': + return 'this run stopped admitting work (it was cancelled, passed its deadline, or too many children went down); no further worker can start' + case 'depth-exceeded': + return "this spawn would exceed the run's maxDepth; a worker at the deepest level cannot start children of its own" + case 'max-live-workers': + return 'the run already has its maximum number of live workers; wait for one to settle (await_event), then spawn again' + case 'duplicate-key': + return 'a worker under this key is still live; wait for it to settle, or use a different key for different work' + case 'key-conflict': + return "this key is already recorded for a different profile or task in this run's journal; use a new key for different work" + case 'invalid-identity': + return 'a keyed spawn needs a complete execution identity to journal, and this profile and task did not produce one; spawn again without a key' + case 'budget-exhausted': { + if (shortfalls === undefined || shortfalls.length === 0) { + return "the conserved pool refused this spawn (budget-exhausted): the run's remaining budget cannot cover this worker's budget" + } + if (shortfalls.some((shortfall) => shortfall.closedByUnknownSpend === true)) { + const closed = shortfalls.find((shortfall) => shortfall.closedByUnknownSpend === true)! + return `the run pool refused this spawn: ${shortfallClause(closed)}; the caller must re-run with a measurable or larger root budget` + } + return `the run pool refused this spawn: ${shortfalls.map(shortfallClause).join('; ')}; or ask the caller for a larger root budget` + } + } +} + /** Build the driver's MCP tools over a live scope. */ export function createCoordinationTools(opts: CoordinationToolsOptions): CoordinationTools { const deliverable = opts.deliverable @@ -3051,21 +3130,22 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin } : { error: res.reason, - // A refusal a driver can ACT on. `usd-unbudgeted` is the one rejection that no - // retry can clear, so it says so: without this, a driver reads "budget" and walks - // its request down until it gives up. - reason: - res.reason === 'usd-unbudgeted' - ? "this run's root budget declares no maxUsd, so a child budget naming maxUsd can never be admitted at any amount — spawn with a budget that omits maxUsd" - : res.reason === 'in-doubt' - ? 'this key has a prior worker recorded as started without a terminal receipt; no replacement was started because that remote worker may still be running — inspect or recover the exact prior execution before retrying' - : // Nothing is exhausted and nothing was cancelled: this run's driver already - // finished and the supervisor is joining. A caller that reaches here is - // working past the end of its own request; the honest report is that the - // stage never started, not that it failed. - res.reason === 'scope-settled' - ? 'this run has already reached its join barrier — its driver returned and the supervisor is settling, so no further worker can be started, joined, or paid for; record this stage as not started' - : `the conserved pool refused this spawn (${String(res.reason)}); the run has no allocation left to give this worker`, + // `usd-unbudgeted` is the one rejection no retry can clear, so it says so: without + // that, a driver reads "budget" and walks its request down until it gives up. + // A refusal a driver can ACT on: each kind says what happened and what to do next. + reason: spawnRefusalReason(res.reason, res.shortfalls, { + usdUnbudgeted: + "this run's root budget declares no maxUsd, so a child budget naming maxUsd can never be admitted at any amount — spawn with a budget that omits maxUsd", + inDoubt: + 'this key has a prior worker recorded as started without a terminal receipt; no replacement was started because that remote worker may still be running — inspect or recover the exact prior execution before retrying', + // Nothing is exhausted and nothing was cancelled: this run's driver already + // finished and the supervisor is joining. A caller that reaches here is working + // past the end of its own request; the honest report is that the stage never + // started, not that it failed. + scopeSettled: + 'this run has already reached its join barrier — its driver returned and the supervisor is settling, so no further worker can be started, joined, or paid for; record this stage as not started', + }), + ...(res.shortfalls === undefined ? {} : { shortfalls: res.shortfalls }), ...(res.reason === 'usd-unbudgeted' ? { hint: diff --git a/src/runtime/index.ts b/src/runtime/index.ts index a555843d..75592732 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -639,6 +639,7 @@ export { type LeakedReservation, type ReservationHolder, type ReservationRejection, + type ReservationShortfall, type ReservationStage, type ReservationTicket, spendFromUsageEvents, diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index 40728566..f2aaf8ec 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -189,6 +189,22 @@ export type BudgetReadout = Readonly<{ * unsatisfiable at any amount and the fix is to budget the root, not to ask for less. */ export type ReservationRejection = 'budget-exhausted' | 'usd-unbudgeted' +/** One budget channel a `budget-exhausted` reservation could not fit, with the amounts that + * decided it. A refusal lists every channel that did not fit (`shortfalls`), so shrinking one + * request is not answered by a second refusal on a channel the caller was never told about + * (observed live: a director whose 100-iteration child was refused spent a probe worker to learn + * the pool still admitted 3). `free` is a snapshot: live reservations return to it as their + * workers settle, and a driver's own metered turns draw tokens and dollars from the same pool + * before its next request arrives, so only `iterations` can be requested at exactly `free`. + * `closedByUnknownSpend` means work with unmeasured usage ran under that enforced limit; the + * channel then refuses every reservation for the rest of the run. */ +export interface ReservationShortfall { + readonly channel: 'tokens' | 'iterations' | 'usd' | `resource:${string}` + readonly requested: number + readonly free: number + readonly closedByUnknownSpend?: true +} + /** State recovered from a prior process before new work is admitted. `committed` is measured spend * already present in the durable journal. Each `uncertainReservation` is a child that was recorded * as started but never recorded as settled: its full declared ceiling is charged conservatively, @@ -294,7 +310,9 @@ export interface BudgetPool { reserve( b: Budget, holder?: ReservationHolder, - ): { ok: true; ticket: ReservationTicket } | { ok: false; reason: ReservationRejection } + ): + | { ok: true; ticket: ReservationTicket } + | { ok: false; reason: ReservationRejection; shortfalls?: readonly ReservationShortfall[] } /** * Name (or rename) who holds an open reservation. Merges into what `reserve` recorded, so a * caller states only what it just learned — the node id admission minted, or the stage the @@ -588,14 +606,14 @@ export function createBudgetPool( function reserve( b: Budget, holder: ReservationHolder = { stage: 'admitted' }, - ): { ok: true; ticket: ReservationTicket } | { ok: false; reason: ReservationRejection } { + ): + | { ok: true; ticket: ReservationTicket } + | { ok: false; reason: ReservationRejection; shortfalls?: readonly ReservationShortfall[] } { assertValidBudget(b, 'reservation budget') for (const [name, state] of resources) { const wanted = b.resources?.[name] if (!wanted) throw new ValidationError(`resource ${name}: child must declare its limit`) if (wanted.unit !== state.unit) throw new ValidationError(`resource ${name}: unit mismatch`) - if (!state.known || wanted.limit > state.remaining) - return { ok: false, reason: 'budget-exhausted' } } for (const name of Object.keys(b.resources ?? {})) { if (!resources.has(name)) @@ -604,18 +622,42 @@ export function createBudgetPool( const wantTokens = b.maxTokens const wantUsd = b.maxUsd ?? 0 const wantIterations = b.maxIterations - if (usdCapped && usdTainted) return { ok: false, reason: 'budget-exhausted' } - // Fail-closed admission: every requested channel must fit the free balance. A - // usd request against an uncapped root is unsatisfiable (the root declared no $). - if (wantTokens > freeTokens) return { ok: false, reason: 'budget-exhausted' } - if (wantIterations > freeIterations) return { ok: false, reason: 'budget-exhausted' } + // Fail-closed admission: every requested channel must fit the free balance. Every channel + // that does not fit is reported, so a caller that shrinks one request is not refused again + // on a second channel it was never told about. + const shortfalls: ReservationShortfall[] = [] + const short = ( + channel: ReservationShortfall['channel'], + requested: number, + free: number, + closedByUnknownSpend = false, + ): void => { + shortfalls.push({ + channel, + requested, + free: closedByUnknownSpend ? 0 : Math.max(0, free), + ...(closedByUnknownSpend ? { closedByUnknownSpend: true as const } : {}), + }) + } + for (const [name, state] of resources) { + const limit = b.resources![name]!.limit + if (!state.known) short(`resource:${name}`, limit, 0, true) + else if (limit > state.remaining) short(`resource:${name}`, limit, state.remaining) + } + if (usdCapped && usdTainted) short('usd', wantUsd, 0, true) + if (wantTokens > freeTokens) short('tokens', wantTokens, freeTokens) + if (wantIterations > freeIterations) short('iterations', wantIterations, freeIterations) // A dollar request against a root that declared no dollar ceiling can never be satisfied at // ANY amount, which is a different fact from an exhausted balance and calls for a different // fix: budget the root, do not retry smaller. Reporting both as `budget-exhausted` invites a // caller to shrink its request forever — observed live, a driver walked its child budget down - // to $0.01 and spent 68k tokens before asking for help. - if (wantUsd > 0 && !usdCapped) return { ok: false, reason: 'usd-unbudgeted' } - if (wantUsd > freeUsd) return { ok: false, reason: 'budget-exhausted' } + // to $0.01 and spent 68k tokens before asking for help. An exhausted channel above still + // takes precedence, as it did before shortfalls were reported. + if (shortfalls.length === 0 && wantUsd > 0 && !usdCapped) { + return { ok: false, reason: 'usd-unbudgeted' } + } + if (usdCapped && !usdTainted && wantUsd > freeUsd) short('usd', wantUsd, freeUsd) + if (shortfalls.length > 0) return { ok: false, reason: 'budget-exhausted', shortfalls } for (const [name, state] of resources) { const amount = b.resources![name]!.limit diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 707f0179..a9c0c285 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -50,6 +50,7 @@ import { createBudgetPool, meterUsageEvent, newUsageTotals, + type ReservationShortfall, type ReservationTicket, spendFromUsageTotals, } from './budget' @@ -704,7 +705,7 @@ export function createScope(args: ScopeArgs): Scope { recovery?: RetainedChildRecovery, ): | { ok: true; handle: Handle; prior?: SpawnPrior } - | { ok: false; reason: SpawnRejection } { + | { ok: false; reason: SpawnRejection; shortfalls?: readonly ReservationShortfall[] } { if (args.signal.aborted) return { ok: false, reason: 'scope-aborted' } // The run reached its join barrier: no later child can be joined, released, or selected over. // Distinct from an abort — nothing cancelled this run (see `closeScopeAdmission`). @@ -813,7 +814,11 @@ export function createScope(args: ScopeArgs): Scope { } if (!reservation.ok) { permit.release() - return { ok: false, reason: reservation.reason } + return { + ok: false, + reason: reservation.reason, + ...(reservation.shortfalls === undefined ? {} : { shortfalls: reservation.shortfalls }), + } } // Resolve the leaf executor through the open registry after both worker and budget admission. diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 2435edc6..01847f77 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -39,6 +39,7 @@ import type { HarnessTranscriptCapture, HarnessTranscriptEvidence } from '../har import type { RetainedInteractiveRunHandle } from '../retained-interactive-types' import type { RetainedRunEffect } from '../retained-run-types' import type { LoopTokenUsage } from '../types' +import type { ReservationShortfall } from './budget' import type { ExecutorProgress, WorkerProgress } from './progress' import type { RetainedPendingCause } from './retained-executor' import type { TraceSource } from './trace-source' @@ -1041,7 +1042,9 @@ export interface Scope { agent: Agent | (() => Agent), task: unknown, opts: SpawnOpts, - ): { ok: true; handle: Handle; prior?: SpawnPrior } | { ok: false; reason: SpawnRejection } + ): + | { ok: true; handle: Handle; prior?: SpawnPrior } + | { ok: false; reason: SpawnRejection; shortfalls?: readonly ReservationShortfall[] } /** ray.wait n=1 over this scope's in-memory live set; resolves as each child settles; * `null` when the live set is empty. */ next(): Promise | null> diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 4e02ee5d..fb091f82 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:caec754baf4cbb48b2799e6264e2fea0be85ff88ad797fd58ecd6e0f02659050", + "digest": "sha256:0f561fbee888ec93525d042d6d8099f2e5f9b0d83b412c31fca4cbd1c68ec158", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.236.0" + "runtimeVersion": "0.237.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:7f960e1005f180f1e102e8f3212133dfb353237036edb815a3dbc439e7515b2e", - "runId": "agent-runtime-0.236.0-proposal-fixture", + "recordDigest": "sha256:d40420c51a75f71cc822c8f128686f8805ae24798f640abd5c4af7a4d15e3d2d", + "runId": "agent-runtime-0.237.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.236.0-proposal-fixture" + "runId": "agent-runtime-0.237.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index cb6fe1b1..5c08dc83 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:7a132aa206675826cba8462e29b77b58c700fa9f967b1f2a1d9403f471e638ee", + "digest": "sha256:e71e0b237230e260aba3bcb6d0512ccc08c28c46fe09c892567ece0fd33a424d", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.236.0" + "runtimeVersion": "0.237.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:d31e5e1a502fdf4e15e02f2fd34138785b138cb4302cfbe7dc367da22fa34ad9", + "recordDigest": "sha256:2a1ba8b4e4e8720c2f5823031c7bc9d4e67ae5e21c6c1090a2981e0d204b91e0", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/kernel/coordination.test.ts b/tests/kernel/coordination.test.ts index 32f18893..fb317377 100644 --- a/tests/kernel/coordination.test.ts +++ b/tests/kernel/coordination.test.ts @@ -335,7 +335,7 @@ describe('coordination tools', () => { expect(await tool(tb, 'spawn_worker').handler({ profile: {}, task: 'go' })).toEqual({ error: 'budget-exhausted', reason: - 'the conserved pool refused this spawn (budget-exhausted); the run has no allocation left to give this worker', + "the conserved pool refused this spawn (budget-exhausted): the run's remaining budget cannot cover this worker's budget", live: 1, freeSlots: null, }) diff --git a/tests/kernel/overspend-settlement.test.ts b/tests/kernel/overspend-settlement.test.ts index a27cf68c..505dec3e 100644 --- a/tests/kernel/overspend-settlement.test.ts +++ b/tests/kernel/overspend-settlement.test.ts @@ -189,7 +189,11 @@ describe('a completed child that overspent its reservation', () => { label: 'next', budget: { maxIterations: 16, maxTokens: 800_000 }, }), - ).toEqual({ ok: false, reason: 'budget-exhausted' }) + ).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'tokens', requested: 800_000, free: 581_873 }], + }) // Replay and the materialized tree read the same record. const [replayed] = await replaySpawnTree(journal, blobs, 'run') diff --git a/tests/kernel/supervise-restart-resource-safety.test.ts b/tests/kernel/supervise-restart-resource-safety.test.ts index 3ea1ffea..6b05eddc 100644 --- a/tests/kernel/supervise-restart-resource-safety.test.ts +++ b/tests/kernel/supervise-restart-resource-safety.test.ts @@ -1159,7 +1159,11 @@ describe('supervision restart and resource safety', () => { if (result.kind !== 'winner') return expect(result.out).toEqual({ downKind: 'down', - second: { ok: false, reason: 'budget-exhausted' }, + second: { + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 1, free: 0, closedByUnknownSpend: true }], + }, }) expect(replacementExecutions).toBe(0) expect(result.spentTotal).toMatchObject({ diff --git a/tests/kernel/supervise.test.ts b/tests/kernel/supervise.test.ts index d6a3b0cd..bfb7efc5 100644 --- a/tests/kernel/supervise.test.ts +++ b/tests/kernel/supervise.test.ts @@ -205,7 +205,11 @@ describe('conserved budget pool', () => { expect(a.ok).toBe(true) // 600 reserved, 400 free; a 500-token child must fail closed (never overcommit). const b = pool.reserve({ maxIterations: 2, maxTokens: 500, label: '' } as Budget) - expect(b).toEqual({ ok: false, reason: 'budget-exhausted' }) + expect(b).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'tokens', requested: 500, free: 400 }], + }) expect(pool.readout().tokensLeft).toBe(400) expect(pool.readout().reservedTokens).toBe(600) }) @@ -234,7 +238,11 @@ describe('conserved budget pool', () => { pool.reserve({ maxIterations: 1, maxTokens: 10, maxUsd: 0.75, label: '' } as Budget).ok, ).toBe(true) const over = pool.reserve({ maxIterations: 1, maxTokens: 10, maxUsd: 0.5, label: '' } as Budget) - expect(over).toEqual({ ok: false, reason: 'budget-exhausted' }) + expect(over).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 0.5, free: 0.25 }], + }) }) it('refunds the unspent remainder on reconcile (Σ conservation)', () => { @@ -276,6 +284,7 @@ describe('conserved budget pool', () => { expect(pool.reserve({ maxIterations: 1, maxTokens: 1 })).toEqual({ ok: false, reason: 'budget-exhausted', + shortfalls: [{ channel: 'tokens', requested: 1, free: 0 }], }) }) @@ -455,6 +464,7 @@ describe('conserved budget pool', () => { expect(pool.reserve({ maxIterations: 1, maxTokens: 10, maxUsd: 0.01 } as Budget)).toEqual({ ok: false, reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 0.01, free: 0 }], }) }) @@ -553,6 +563,7 @@ describe('conserved budget pool', () => { expect(pool.reserve({ maxIterations: 1, maxTokens: 10 } as Budget)).toEqual({ ok: false, reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 0, free: 0, closedByUnknownSpend: true }], }) }) @@ -723,6 +734,10 @@ describe('conserved budget pool', () => { expect(pool.reserve({ maxIterations: 1, maxTokens: 1 } as Budget)).toEqual({ ok: false, reason: 'budget-exhausted', + shortfalls: [ + { channel: 'usd', requested: 0, free: 0, closedByUnknownSpend: true }, + { channel: 'iterations', requested: 1, free: 0 }, + ], }) expect(() => pool.assertNoOpenTickets()).not.toThrow() }) @@ -1205,7 +1220,11 @@ describe('equal-k by construction', () => { label: 'after-unknown-cost', budget: { maxIterations: 1, maxTokens: 1 }, }), - ).toEqual({ ok: false, reason: 'budget-exhausted' }) + ).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 0, free: 0, closedByUnknownSpend: true }], + }) }) // ── The cost-event × declared-ceiling matrix ────────────────────────────────── @@ -1311,7 +1330,11 @@ describe('equal-k by construction', () => { label: 'after-unknown', budget: { maxIterations: 1, maxTokens: 1 }, }), - ).toEqual({ ok: false, reason: 'budget-exhausted' }) + ).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'usd', requested: 0, free: 0, closedByUnknownSpend: true }], + }) }) it('records a priced child on an UNCAPPED root as observed-but-unbudgeted dollars', async () => { @@ -1677,7 +1700,14 @@ describe('reactive scope', () => { budget: { maxIterations: 1, maxTokens: 10 }, label: 'b', }) - expect(overflow).toEqual({ ok: false, reason: 'budget-exhausted' }) + expect(overflow).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [ + { channel: 'tokens', requested: 10, free: 0 }, + { channel: 'iterations', requested: 1, free: 0 }, + ], + }) }) it('abort mid-flight reaps the live child (down, no throw)', async () => { diff --git a/tests/runtime/spawn-refusal-shortfall.test.ts b/tests/runtime/spawn-refusal-shortfall.test.ts new file mode 100644 index 00000000..179a622d --- /dev/null +++ b/tests/runtime/spawn-refusal-shortfall.test.ts @@ -0,0 +1,192 @@ +/** + * A refused spawn says which budget channels fell short and by how much. + * + * Measured 2026-09-16 on a Discovery director placed on the Tangle sandbox: its first research + * child asked for 100 iterations against a 60-iteration pool and got back "the conserved pool + * refused this spawn (budget-exhausted); the run has no allocation left to give this worker". + * Nothing in that told it the pool still admitted 60, so it spent a throwaway probe worker with 3 + * iterations to find out, then moved its research to local processes. + */ + +import { describe, expect, it } from 'vitest' +import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { spawnRefusalReason } from '../../src/mcp/tools/coordination' +import { createBudgetPool } from '../../src/runtime/supervise/budget' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import type { + Agent, + AgentSpec, + Executor, + ExecutorResult, + Scope, + SpawnRejection, +} from '../../src/runtime/supervise/types' +import { testAgentProfile } from '../kernel/test-agent-profile' + +const pinned = { + usdUnbudgeted: 'usd-unbudgeted text', + inDoubt: 'in-doubt text', + scopeSettled: 'scope-settled text', +} + +/** A child that would run if admitted. The refusal must come before its executor is touched. */ +function childThatMustNotRun(): Agent { + const executor: Executor = { + runtime: 'router', + execute: async (): Promise> => { + throw new Error('a refused child must not execute') + }, + teardown: async () => ({ destroyed: true }), + } + const executorSpec: AgentSpec = { + profile: testAgentProfile('child'), + harness: null, + executor: executor as Executor, + } + return { name: 'child', act: async () => 'never', executorSpec } as Agent< + unknown, + string | undefined + > +} + +describe('budget pool refusal', () => { + it('names the channel, the request, and what is still free', () => { + const pool = createBudgetPool({ maxIterations: 60, maxTokens: 3_000_000 }, 0) + expect(pool.reserve({ maxIterations: 100, maxTokens: 800_000 })).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'iterations', requested: 100, free: 60 }], + }) + // A request sized from `free` on iterations is admitted. + expect(pool.reserve({ maxIterations: 60, maxTokens: 800_000 }).ok).toBe(true) + // With the pool fully reserved, the free balance reads 0, never negative. + expect(pool.reserve({ maxIterations: 1, maxTokens: 1 })).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'iterations', requested: 1, free: 0 }], + }) + }) + + it('lists every channel that does not fit, not only the first', () => { + const pool = createBudgetPool({ maxIterations: 60, maxTokens: 3_000_000, maxUsd: 2 }, 0) + expect(pool.reserve({ maxIterations: 100, maxTokens: 4_000_000, maxUsd: 5 })).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [ + { channel: 'tokens', requested: 4_000_000, free: 3_000_000 }, + { channel: 'iterations', requested: 100, free: 60 }, + { channel: 'usd', requested: 5, free: 2 }, + ], + }) + }) + + it('keeps usd-unbudgeted for a dollar request against an uncapped root', () => { + const uncapped = createBudgetPool({ maxIterations: 10, maxTokens: 1_000 }, 0) + expect(uncapped.reserve({ maxIterations: 1, maxTokens: 10, maxUsd: 1 })).toEqual({ + ok: false, + reason: 'usd-unbudgeted', + }) + }) +}) + +describe('spawn refusal reaches the driver', () => { + it('carries the shortfalls from the pool through scope.spawn', async () => { + let refusal: ReturnType['spawn']> | undefined + const settled = await createSupervisor().run( + { + name: 'root', + act: async (_task: unknown, scope: Scope) => { + refusal = scope.spawn(childThatMustNotRun(), 'go', { + budget: { maxIterations: 100, maxTokens: 10_000 }, + label: 'too-big', + }) + return undefined + }, + } as Agent, + 'task', + { + budget: { maxIterations: 60, maxTokens: 100_000 }, + runId: 'spawn-refusal-shortfall', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + now: () => 0, + }, + ) + expect(settled.fleetYield?.spawned ?? 0).toBe(0) + expect(refusal).toEqual({ + ok: false, + reason: 'budget-exhausted', + shortfalls: [{ channel: 'iterations', requested: 100, free: 60 }], + }) + }) + + it('tells the driver the exact iteration ceiling', () => { + expect( + spawnRefusalReason( + 'budget-exhausted', + [{ channel: 'iterations', requested: 100, free: 58 }], + pinned, + ), + ).toBe( + 'the run pool refused this spawn: iterations has 58 free (this spawn asked for budget.maxIterations 100); budget.maxIterations at most 58 fits; or ask the caller for a larger root budget', + ) + }) + + it('does not promise tokens or dollars a driver turn will spend first', () => { + // A driver's own turn is metered from the same pool before its retry reaches admission, so a + // retry at exactly `free` is refused again (reproduced in review of #1271). The advice must + // not name that number as a request that fits. + for (const channel of ['tokens', 'usd'] as const) { + const text = spawnRefusalReason( + 'budget-exhausted', + [{ channel, requested: 20_000, free: 9_850 }], + pinned, + ) + expect(text, channel).toMatch(/ask for well under 9850/u) + expect(text, channel).not.toMatch(/at most 9850 fits/u) + } + }) + + it('names every short channel, and a closed channel as closed for the run', () => { + const both = spawnRefusalReason( + 'budget-exhausted', + [ + { channel: 'tokens', requested: 900, free: 0 }, + { channel: 'resource:gpuSeconds', requested: 30, free: 12 }, + ], + pinned, + ) + expect(both).toMatch(/tokens has nothing free \(this spawn asked for budget\.maxTokens 900\)/u) + expect(both).toMatch(/budget\.resources\.gpuSeconds\.limit at most 12 fits/u) + expect( + spawnRefusalReason( + 'budget-exhausted', + [{ channel: 'usd', requested: 0, free: 0, closedByUnknownSpend: true }], + pinned, + ), + ).toMatch(/admits no further spawn at any budget/u) + }) + + it('does not call a non-budget refusal an empty pool', () => { + const kinds: SpawnRejection[] = [ + 'depth-exceeded', + 'duplicate-key', + 'invalid-identity', + 'key-conflict', + 'max-live-workers', + 'scope-aborted', + ] + for (const kind of kinds) { + const text = spawnRefusalReason(kind, undefined, pinned) + expect(text, kind).not.toMatch(/pool|allocation/u) + expect(text.length, kind).toBeGreaterThan(20) + } + expect(spawnRefusalReason('max-live-workers', undefined, pinned)).not.toMatch(/cancel/u) + expect(spawnRefusalReason('invalid-identity', undefined, pinned)).toMatch(/without a key/u) + expect(spawnRefusalReason('usd-unbudgeted', undefined, pinned)).toBe('usd-unbudgeted text') + expect(spawnRefusalReason('in-doubt', undefined, pinned)).toBe('in-doubt text') + expect(spawnRefusalReason('scope-settled', undefined, pinned)).toBe('scope-settled text') + }) +})