diff --git a/CHANGELOG.md b/CHANGELOG.md index 909e42ce..23afa8ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,50 @@ blobs)`. Consequences, all deliberate: Settlements recorded before this release carry no `harnessTranscript` on the record, which is not the same fact as a recorded `unavailable`, and stays absent rather than defaulting. +**A released retained child now settles (#1247).** A child whose provider execution was RETAINED +— admitted durably, result read lost — settles `down` with its cursor slot open and its +reservation reconciled at the floor it had streamed (0.230.0, #1190), so that a later process +could resume the run and reconcile the paid execution. On the pursuit path no later process ever +comes: `supervisePursuit` writes a settle record that refuses re-entry and forces +`retainedAtSettlement: 'release'`, so every one of these children is released at root settlement +and none is ever recovered — by construction, not by accident. Measured 2026-09-15: on +`capability-per-parameter-cpp-glm-20260915c`, 0 of 35 reconciled children ever settled, and 185 +of 223 lost sandbox children across 385 runs stopped at `reconciled`. Each one read as +`never-settled` to every journal reader and was charged its ceiling, while the pool had committed +the floor; Lab told them apart from ordinary downs by matching the reason string. + +- The release sweep now writes the node's terminal record after the `environment-teardown` + receipt, once the executor confirms teardown: the settlement the driver received (reason, + `infra`, `trace`, `harnessTranscript`, `outRef`, `providerModel`), under the seq the driver + saw, marked `retainedExecution: 'released'`. Its `spent` is the node's child-work component + of the reconcile the pool committed — the streamed floor for a leaf, `accounting().reported` + for a recursive executor — never the reservation. A cancelled retained child keeps the + `cancelled` kind and its `source`. The overspend a retained reconcile returned, which the + open-slot surfaces withhold, is carried on this record, because when the run releases the node + the floor is its final charge. A refused release (`destroyed: false`, or an executor whose + teardown will not confirm) writes nothing: the environment may still exist, so the slot stays + open and the node stays in `teardownUnconfirmed`. +- `retainedExecution: 'pending' | 'released'` (new `RetainedExecutionState`) is stated on + `Settled`'s down arm (`'pending'` at the reconcile, `'released'` on replay of a closed node), + on the journal's `settled`/`cancelled` records (`'released'` only), on `NodeSnapshot` from + both the live and the materialized tree, on the `agent.child` payload (a second event per + released node, carrying `releasedAt`), and on `PursuitNodeProjection`. A reader never splits + the population on `reason` text again. `PursuitStatus` stays three-valued. +- `SupervisedResult` gains `fleetYield` on every arm (new `FleetYield`): `spawned`, `done`, + `down`, `cancelled`, `neverSettled`, `releasedUnrecovered`, counted by node id across the + whole journal forest, with `spawned === done + down + cancelled + neverSettled` asserted. + Always present; zeros are facts. It rides into `result.json`. `spendGaps` stays root-tree + scoped, so the two disagree on a nested run by design. A no-winner's `downCount` is unchanged: + the breaker skips released records, so a retained child is counted in `fleetYield.down` and + never in `downCount` — as it never was. +- `spentTotal.tokensKnown`/`usdKnown` stay `false` on a run with a released child: its record + carries the unknown-flagged floor, so the gap changes kind from `never-settled` (a ceiling) to + `unreported` (a floor), not presence. The one exception is a ticket already reconciled at a + measured terminal spend (a persistence failure after the artifact landed), whose released record + is known. The pool half — the refund at the reconcile (#1190) — is unchanged. +- A crash between the receipt and the terminal record leaves an open slot beside a destroyed + environment; the next process cannot self-heal it and this release does not try. + ## 0.231.1 Allow Sandbox 0.40.x beside Runtime without a consumer dependency override. diff --git a/api-surface.json b/api-surface.json index bd96ceff..db23d299 100644 --- a/api-surface.json +++ b/api-surface.json @@ -505,7 +505,7 @@ "PursuitNodeCost": "type 2ae084b090ba", "PursuitNodePlacement": "type 744e7b45dc01", "PursuitNodePlatform": "type 1d5dc2e2dd89", - "PursuitNodeProjection": "type 85fb852e6043", + "PursuitNodeProjection": "type 6d2b6c593df9", "PursuitNodeTiming": "type 8cb37b8dca25", "PursuitNodeUsage": "type 951e957775ab", "PursuitProjection": "type 8d4e8a08d707", @@ -944,6 +944,7 @@ "FinalizeContext": "type e45cea6eacef", "FinalizerSettled": "type c525a06427fc", "FlatWidenGate": "type 825b8b585b72", + "FleetYield": "type c6e215cee2d9", "ForkCapableBox": "type 87d9804bb9f3", "ForkRequest": "type 7cd1fb62a8d3", "GitWorkspaceOptions": "type 47832f185060", @@ -1043,7 +1044,7 @@ "NoWinnerError": "type adcf245eb125", "NodeExecutionIdentity": "type 76b27bc384d1", "NodeId": "type ba5552a40a12", - "NodeSnapshot": "type a9b2602d5c10", + "NodeSnapshot": "type 7a1dd9177e44", "NodeStatus": "type 2073c1387091", "Observation": "type c276b53b9eb0", "ObservationAnalysis": "type b23fcc6368fb", @@ -1140,6 +1141,7 @@ "ResultBlobStore": "type 69e6e03b75ab", "ResumedKeyState": "type 018905da27e4", "ResumedWork": "type 57b7c709155a", + "RetainedExecutionState": "type 0bb1e591e45f", "RetainedInteractiveAdmission": "type ecfa4c1fe733", "RetainedInteractiveAdmissionHook": "type bfe9af8b2361", "RetainedInteractiveEnvironmentAdmission": "type b1361c236b09", @@ -1209,13 +1211,13 @@ "SessionCapableBox": "type 9db08b40cab8", "SessionMessageLike": "type 726215634de7", "SessionTraceBox": "type b8d989093064", - "Settled": "type 4694c79a44d1", + "Settled": "type 24da40c1d786", "ShapeBudget": "type a1cb1a8d0995", "ShapeContext": "type a266c1b7e746", "ShapeRegistry": "type b567a5be55ec", "Shell": "type c676fe970f79", "ShotSpec": "type bf645fd74234", - "SpawnEvent": "type e74c7b38c734", + "SpawnEvent": "type f9c5663b41cd", "SpawnForest": "type 7308f34da226", "SpawnForestEvent": "type 230e369c6548", "SpawnForestInDoubtNode": "type 4e4bddd6a7d9", @@ -1263,7 +1265,7 @@ "SuperviseRegistryTable": "type cc1468cd50c1", "SuperviseSurfaceOptions": "type 8a7daaf98896", "SuperviseSurfaceResult": "type 2c378dbc3193", - "SupervisedResult": "type 924803bb65bd", + "SupervisedResult": "type e23388c4e1ca", "Supervisor": "type 7d9aff9cd744", "SupervisorAgentDeps": "type b2683b00f9e6", "SupervisorCleanupReceipt": "type d862eb60266d", @@ -1626,7 +1628,7 @@ "streamAgentTurn": "value d21f23205538", "structuralRollout": "value fc852ab503f9", "sumSandboxUsage": "value 368f5c1a5123", - "supervise": "value ec5717d6f3aa", + "supervise": "value 0f64939f8d20", "superviseDispatch": "value 8f93690a9eb5", "superviseSurface": "value 730f753cfb58", "supervisorAgent": "value 880407b0dab3", @@ -1980,7 +1982,7 @@ "loadAgentImprovementProposalFixture": "value 2f33e49d4be9", "loadAgentProfileImprovementFixture": "value 3bf4543ae6f7", "runGraphWithTestBrain": "value 5d9fea087acc", - "superviseWithTestBrain": "value 68eed805ecfe", + "superviseWithTestBrain": "value 837346617a33", "supervisorAgentWithTestBrain": "value 5c814f8dd593" }, "./tool-loop": { diff --git a/docs/api/durable.md b/docs/api/durable.md index 64a24350..80848f0c 100644 --- a/docs/api/durable.md +++ b/docs/api/durable.md @@ -1030,6 +1030,23 @@ Content-addressed pointer to this node's persisted tool trace, or why there is n > `readonly` `optional` **infra?**: `boolean` +##### retainedExecution? + +> `readonly` `optional` **retainedExecution?**: [`RetainedExecutionState`](runtime.md#retainedexecutionstate) + +Recorded by Runtime on the `agent.child` payload: `'pending'` at a retained child's + settlement, `'released'` when root settlement destroyed its environment without recovery. + The status stays `down` — the split is a sibling fact, not a fourth status. A second + `agent.child` for one node is already how a live-recovered child flips down→done, so the + fold overwrites in observed order; that event's `settledAt` is the original settlement, so + `settledAt` and `timing` do not move. + +##### releasedAt? + +> `readonly` `optional` **releasedAt?**: `number` + +When the release sweep closed a retained node's slot; absent unless `'released'`. + ##### budgetViolation? > `readonly` `optional` **budgetViolation?**: [`BudgetViolation`](runtime.md#budgetviolation-3) @@ -2734,7 +2751,7 @@ Read the most recent failure record, or `undefined` when the directory holds non ### supervisePursuit() -> **supervisePursuit**(`profile`, `task`, `opts`): `Promise`\<[`SupervisedPursuitResult`](#supervisedpursuitresult)\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\>\> +> **supervisePursuit**(`profile`, `task`, `opts`): `Promise`\<[`SupervisedPursuitResult`](#supervisedpursuitresult)\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\>\> One-call durable pursuit execution over the canonical `supervise()` kernel. @@ -2771,7 +2788,7 @@ An abandoned `supervise.lock.guard` requires removal after confirming no lock mu #### Returns -`Promise`\<[`SupervisedPursuitResult`](#supervisedpursuitresult)\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\>\> +`Promise`\<[`SupervisedPursuitResult`](#supervisedpursuitresult)\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\>\> *** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index db741600..9db3b744 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -423,7 +423,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 167 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 954 exports. +Import from `@tangle-network/agent-runtime/kernel` — 956 exports. | Symbol | Kind | Summary | |---|---|---| @@ -869,6 +869,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 954 exports. | `FanoutSynthesis` | interface | How a fanout's synthesis child is built + read. `synthesisTask` projects the drained child | | `FinalizeContext` | interface | What a finalizer gets to decide with. `delivered` is the ONLY output material; `allSettled` | | `FinalizerSettled` | interface | One settled worker as the finalizer sees it — the ledger row (structural fields only). | +| `FleetYield` | interface | How this run's spawned CHILDREN ended, counted by node id off the complete journal FOREST at | | `ForkCapableBox` | interface | Loop-side widening of the legacy checkpoint fork method. | | `GraphNode` | interface | A graph node: an id and a canonical `AgentProfile`. The profile is the ONLY way a node is | | `Handle` | interface | A live child handle. `abort()` is defined over the ACQUIRE lifecycle: it chains into | @@ -1155,6 +1156,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 954 exports. | `ReservationStage` | type | Where in the spawn lifecycle a reservation was last seen. `admitted` is the window between | | `ResolveDriveHarness` | type | Resolve an external harness for one exact Runtime-owned manager identity. | | `ResolveSupervisorTools` | type | Product policy for the tools one exact supervisor node may call. Resolved once per node. | +| `RetainedExecutionState` | type | The recorded fate of a child whose provider execution was RETAINED: admitted durably, with no | | `RetainedInteractiveAdmission` | type | Durable records for one exact native coding-agent process. | | `RetainedInteractiveAdmissionHook` | type | Persist each exact interactive record before the runtime proceeds. | | `RetainedInteractiveEnvironmentInput` | type | Environment and exact AgentProfile used to start one native coding-agent process. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 6b49964a..1f44127e 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -1158,6 +1158,19 @@ Present once a settled node's measured spend exceeded its reservation. [`NodeSnapshot`](#nodesnapshot).[`budgetViolation`](#budgetviolation-2) +##### retainedExecution? + +> `readonly` `optional` **retainedExecution?**: [`RetainedExecutionState`](#retainedexecutionstate) + +Present on a retained child: `'pending'` while its cursor slot is open, `'released'` once + the release sweep closed it. The live view (`makeTreeView`) and the journal view + (`materializeTreeView`) state the same fact, so a settle record's `tree` answers the + retained-vs-down question without the observer journal. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`retainedExecution`](#retainedexecution-1) + *** ### SpawnForestInDoubtNode @@ -15006,10 +15019,11 @@ An explicit run deadline always wins. Omit/`0` = immediate teardown. > `readonly` `optional` **retainedAtSettlement?**: `"release"` \| `"keep"` What root settlement does with provider environments that settled children still hold for a -retained execution: `'release'` them with one `environment-teardown` receipt each, or `'keep'` -them for a later call that resumes this run. Default: `'keep'` with `runDir` (re-running the -same `runDir` and `runId` resumes), `'release'` without it (nothing can resume an in-memory -run). See `SupervisorOpts.retainedAtSettlement`. +retained execution: `'release'` them with one `environment-teardown` receipt each and close +each released child's cursor slot with a terminal record marked `retainedExecution: +'released'`, or `'keep'` them for a later call that resumes this run. Default: `'keep'` with +`runDir` (re-running the same `runDir` and `runId` resumes), `'release'` without it (nothing +can resume an in-memory run). See `SupervisorOpts.retainedAtSettlement`. ###### Inherited from @@ -19157,10 +19171,11 @@ An explicit run deadline always wins. Omit/`0` = immediate teardown. > `readonly` `optional` **retainedAtSettlement?**: `"release"` \| `"keep"` What root settlement does with provider environments that settled children still hold for a -retained execution: `'release'` them with one `environment-teardown` receipt each, or `'keep'` -them for a later call that resumes this run. Default: `'keep'` with `runDir` (re-running the -same `runDir` and `runId` resumes), `'release'` without it (nothing can resume an in-memory -run). See `SupervisorOpts.retainedAtSettlement`. +retained execution: `'release'` them with one `environment-teardown` receipt each and close +each released child's cursor slot with a terminal record marked `retainedExecution: +'released'`, or `'keep'` them for a later call that resumes this run. Default: `'keep'` with +`runDir` (re-running the same `runDir` and `runId` resumes), `'release'` without it (nothing +can resume an in-memory run). See `SupervisorOpts.retainedAtSettlement`. ##### resolveDriveHarness? @@ -22471,6 +22486,15 @@ Present on terminal executor nodes; legacy records carry an explicit unavailable Present once a settled node's measured spend exceeded its reservation. +##### retainedExecution? + +> `readonly` `optional` **retainedExecution?**: [`RetainedExecutionState`](#retainedexecutionstate) + +Present on a retained child: `'pending'` while its cursor slot is open, `'released'` once + the release sweep closed it. The live view (`makeTreeView`) and the journal view + (`materializeTreeView`) state the same fact, so a settle record's `tree` answers the + retained-vs-down question without the observer journal. + *** ### TreeView @@ -22785,7 +22809,11 @@ the paid execution inside it. - `'release'`: the run will not be resumed, so nothing would ever reconcile or release those environments. The join barrier releases each one and journals an `environment-teardown` receipt per environment. Measured 2026-09-11: four settled runs held 18 of a 60-slot - Sandbox fleet for 19 to 37 hours without it. + Sandbox fleet for 19 to 37 hours without it. It then closes each released child's cursor + slot with a terminal record marked `retainedExecution: 'released'`, so `spendGaps` names + it `unreported` (a floor) rather than `never-settled` (a ceiling) and + `fleetYield.releasedUnrecovered` counts it; a refused release leaves the slot open and + the node in `teardownUnconfirmed`. - `'keep'`: a later process may resume this run, so the environments stay for its recovery. Default: `'keep'` when `resume` is true (a durable run a later process may continue), else @@ -22930,6 +22958,57 @@ The spawn label, when the node's `spawned` event is in this journal tree. *** +### FleetYield + +How this run's spawned CHILDREN ended, counted by node id off the complete journal FOREST at +root settlement — after the join barrier and the release sweep, so the terminal records the +sweep wrote are included, and forest-wide so a director's grandchildren count (the population +the observer projection shows). Invariant: `spawned === done + down + cancelled + neverSettled`. +Distinct from a no-winner's `downCount`, the breaker's tally of ordinary down settlements: a +released record is the driver's earlier down re-stated, so the breaker skips it and +`downCount` never includes a retained child, while `down` here does. `spendGaps` stays +root-tree scoped, so on a nested run the two disagree by design. + +#### Properties + +##### spawned + +> `readonly` **spawned**: `number` + +Every `spawned` record with a parent; the run root and an owned tree's re-rooted owner copy + are exempt. + +##### done + +> `readonly` **done**: `number` + +##### down + +> `readonly` **down**: `number` + +`settled` records with `status: 'down'`, released records included. + +##### cancelled + +> `readonly` **cancelled**: `number` + +##### neverSettled + +> `readonly` **neverSettled**: `number` + +Spawned with no terminal record: a crash-orphaned child, a refused release, or a retained + executor with nothing to release. Named after `SpendGap`'s `never-settled`. + +##### releasedUnrecovered + +> `readonly` **releasedUnrecovered**: `number` + +Terminal records marked `retainedExecution: 'released'` — a subset of `down + cancelled`. + A nested manager whose OWN retained execution was released counts here beside the + grandchildren it released, because its execution was destroyed unrecovered too. + +*** + ### BudgetOverspend One channel on which a settled reservation's measured spend exceeded what it reserved. @@ -28498,7 +28577,7 @@ recovery before a replacement can run. ### Settled -> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} +> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `settledAt?`: `number`; `seq`: `number`; \} A settled child, delivered by `scope.next()`. `seq` is the monotonic cursor order `next()` yielded this settlement (B2) — NOT wall-clock — and replay delivers strictly @@ -28582,7 +28661,7 @@ Epoch ms parsed from the durable settlement record when available. ##### Type Literal -\{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `settledAt?`: `number`; `seq`: `number`; \} +\{ `kind`: `"down"`; `handle`: [`Handle`](#handle-3)\<`Out`\>; `reason`: `string`; `outRef?`: `string`; `infra`: `boolean`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: [`RetainedExecutionState`](#retainedexecutionstate); `settledAt?`: `number`; `seq`: `number`; \} ###### kind @@ -28636,6 +28715,17 @@ Partial provider model evidence survives an aborted or failed execution. Present when the spend reconciled for this child exceeded its reservation. +###### retainedExecution? + +> `optional` **retainedExecution?**: [`RetainedExecutionState`](#retainedexecutionstate) + +Present only when this child's provider execution was RETAINED (see + `RetainedExecutionState`); absent on an ordinary down. `'pending'` on the settlement the + driver receives at the reconcile; `'released'` on the replayed settlement of a node the + release sweep closed. The driver and every replay reader see the fact the journal + states, so a reader never splits this population on `reason` text — which is identical + on every one of these children. + ###### settledAt? > `optional` **settledAt?**: `number` @@ -28650,7 +28740,7 @@ Epoch ms parsed from the durable settlement/cancellation record when available. ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](#budget-18); `runtime`: [`Runtime`](#runtime-7); `ownedTreeRoot?`: [`NodeId`](#nodeid-6); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `profileRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-input"`; `id`: [`NodeId`](#nodeid-6); `taskRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-admitted"`; `id`: [`NodeId`](#nodeid-6); `admission`: [`RetainedRunAdmission`](#retainedrunadmission); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-result"`; `outcome?`: `Pick`\<`AgentTurnResult`, `"success"` \| `"error"`\>; `id`: [`NodeId`](#nodeid-6); `outRef`: `string`; `spent`: [`Spend`](#spend); `verdict?`: `DefaultVerdict`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-6); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-6); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"node-inputs-resolved"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `instance`: `string`; `inputRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge-verdict"`; `id`: [`NodeId`](#nodeid-6); `edge`: `string`; `fired`: `boolean`; `sourceStatus`: `"done"` \| `"down"` \| `"invalid"`; `capped?`: `boolean`; `inputRef?`: `string`; `toInstance?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"join-state"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `rule`: `"all"` \| `"any"` \| `"any_failed"` \| `"all_done"`; `satisfiedBy`: `ReadonlyArray`\<`string`\>; `consumedPending`: `ReadonlyArray`\<`string`\>; `instance`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-6); `by`: `"fired"` \| `"timeout"` \| `"cancelled"` \| `"expired"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `accountingOnly?`: `true`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"progress"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"teardown-unconfirmed"`; `id`: [`NodeId`](#nodeid-6); `label`: `string`; `runtime`: [`Runtime`](#runtime-7); `status`: [`NodeStatus`](#nodestatus); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"environment-teardown"`; `id`: [`NodeId`](#nodeid-6); `provider`: `string`; `environmentId`: `string`; `destroyed`: `boolean`; `detail?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge"`; `id`: [`NodeId`](#nodeid-6); `edge`: \{ `kind`: `"delegates"` \| `"analyzes"` \| `"data"`; `from`: `string`; `to`: `string`; `directive?`: `string`; `port?`: `string`; \}; `traversal`: `number`; `outcome`: `"delivered"` \| `"stripped"` \| `"empty"` \| `"unpropagated"`; `continuity?`: `"fresh"` \| `"resume"` \| `"steer"`; `bytes`: `number`; `reason?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"trace-unpropagated"`; `id`: [`NodeId`](#nodeid-6); `expectedTraceId`: `string`; `backend`: `string`; `reason`: `"no-env-channel"` \| `"no-worker-process"` \| `"caller-omitted"`; `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](#budget-18); `runtime`: [`Runtime`](#runtime-7); `ownedTreeRoot?`: [`NodeId`](#nodeid-6); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `profileRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-input"`; `id`: [`NodeId`](#nodeid-6); `taskRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-admitted"`; `id`: [`NodeId`](#nodeid-6); `admission`: [`RetainedRunAdmission`](#retainedrunadmission); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-result"`; `outcome?`: `Pick`\<`AgentTurnResult`, `"success"` \| `"error"`\>; `id`: [`NodeId`](#nodeid-6); `outRef`: `string`; `spent`: [`Spend`](#spend); `verdict?`: `DefaultVerdict`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-6); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-6); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"node-inputs-resolved"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `instance`: `string`; `inputRef`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge-verdict"`; `id`: [`NodeId`](#nodeid-6); `edge`: `string`; `fired`: `boolean`; `sourceStatus`: `"done"` \| `"down"` \| `"invalid"`; `capped?`: `boolean`; `inputRef?`: `string`; `toInstance?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"join-state"`; `id`: [`NodeId`](#nodeid-6); `node`: `string`; `rule`: `"all"` \| `"any"` \| `"any_failed"` \| `"all_done"`; `satisfiedBy`: `ReadonlyArray`\<`string`\>; `consumedPending`: `ReadonlyArray`\<`string`\>; `instance`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-6); `parent?`: [`NodeId`](#nodeid-6); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-6); `by`: `"fired"` \| `"timeout"` \| `"cancelled"` \| `"expired"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `accountingOnly?`: `true`; `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"progress"`; `id`: [`NodeId`](#nodeid-6); `spend`: [`Spend`](#spend); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"reconciled"`; `id`: [`NodeId`](#nodeid-6); `spent`: [`Spend`](#spend); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"teardown-unconfirmed"`; `id`: [`NodeId`](#nodeid-6); `label`: `string`; `runtime`: [`Runtime`](#runtime-7); `status`: [`NodeStatus`](#nodestatus); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"environment-teardown"`; `id`: [`NodeId`](#nodeid-6); `provider`: `string`; `environmentId`: `string`; `destroyed`: `boolean`; `detail?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"edge"`; `id`: [`NodeId`](#nodeid-6); `edge`: \{ `kind`: `"delegates"` \| `"analyzes"` \| `"data"`; `from`: `string`; `to`: `string`; `directive?`: `string`; `port?`: `string`; \}; `traversal`: `number`; `outcome`: `"delivered"` \| `"stripped"` \| `"empty"` \| `"unpropagated"`; `continuity?`: `"fresh"` \| `"resume"` \| `"steer"`; `bytes`: `number`; `reason?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"trace-unpropagated"`; `id`: [`NodeId`](#nodeid-6); `expectedTraceId`: `string`; `backend`: `string`; `reason`: `"no-env-channel"` \| `"no-worker-process"` \| `"caller-omitted"`; `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -28886,7 +28976,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ##### Type Literal -\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-6); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} ###### kind @@ -28951,6 +29041,21 @@ Whether this child's harness transcript survived, or the named reason it did not Present when the reconciled spend exceeded the reservation, on either status. +###### retainedExecution? + +> `optional` **retainedExecution?**: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\> + +Written only by the release sweep, on the same tree, after every `environment-teardown` + receipt for the node reads `destroyed: true` and the executor's own teardown confirmed. + `spent` is this node's child-work component of the reconcile the pool committed — for + a leaf the streamed floor itself, for a recursive executor its `accounting().reported` + split with the remainder on its `metered` records — never the reservation ceiling. + `reason`/`infra`/`trace`/`harnessTranscript`/`outRef`/`providerModel` are the + settlement the driver received, verbatim; `seq` is the cursor seq stamped on that + delivery, so replay yields it at the position the driver saw it; `at` is the settlement + instant, and the release instant is on the receipt immediately before it. Typed so a + `'pending'` can never be journaled: the journal states that as `reconciled`. + ###### seq > **seq**: `number` @@ -28963,7 +29068,7 @@ Present when the reconciled spend exceeded the reservation, on either status. ##### Type Literal -\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-6); `reason`: `string`; `source?`: `string`; `infra?`: `boolean`; `spent?`: [`Spend`](#spend); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `trace?`: [`WorkerTraceEvidence`](#workertraceevidence); `harnessTranscript?`: [`HarnessTranscriptEvidence`](#harnesstranscriptevidence); `outRef?`: `string`; `budgetViolation?`: [`BudgetViolation`](#budgetviolation-3); `retainedExecution?`: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\>; `seq`: `number`; `at`: `string`; \} ###### kind @@ -29012,6 +29117,13 @@ The child's harness transcript receipt, when the executor could still be read at > `optional` **budgetViolation?**: [`BudgetViolation`](#budgetviolation-3) +###### retainedExecution? + +> `optional` **retainedExecution?**: `Extract`\<[`RetainedExecutionState`](#retainedexecutionstate), `"released"`\> + +As on `settled`: a retained child that was cancelled settles `cancelled`, and the one + builder writes whichever kind the settlement had. + ###### seq > **seq**: `number` @@ -29335,7 +29447,9 @@ A retained child's reservation was reconciled at the child-work floor its execut observed, while its cursor slot stays OPEN so a resume can recover the execution. It stands in for the `settled` record an open node cannot carry: cost readers and a restored pool charge this floor for the node instead of its declared ceiling, and a later `settled` or - `cancelled` record for the same node supersedes it. A driver's own inference travels on its + `cancelled` record for the same node supersedes it. That record has two writers: a + resumed process's recovered settlement, or the release sweep's terminal record marked + `retainedExecution: 'released'`. A driver's own inference travels on its `metered` record as on every other path, so `reconciled + metered` is what the pool committed. Its `seq` lives outside the cursor-uniqueness namespace. @@ -29421,7 +29535,11 @@ One provider environment a settled retained-pending child held, released at root the environment its executor kept for recovery could never be recovered; this is the receipt of the supervisor's release, one per environment, naming the provider's own id so a fleet listing can be reconciled against it. A `destroyed: false` receipt carries why - in `detail`, and the node is then also journaled as `teardown-unconfirmed`. + in `detail`, and the node is then also journaled as `teardown-unconfirmed`. When every + receipt for a node is `destroyed: true` and the executor confirms teardown, the node's + terminal `settled`/`cancelled` record with `retainedExecution: 'released'` follows on + the same tree and closes the cursor slot; a `destroyed: false` receipt or an unconfirmed + teardown leaves the slot open because the environment may still exist. Informational: replay, `materializeTreeView`, and cost readers skip it, and its `seq` is per node, outside the cursor-uniqueness namespace. @@ -29617,9 +29735,34 @@ The accounting channels a usage gap leaves incomplete. *** +### RetainedExecutionState + +> **RetainedExecutionState** = `"pending"` \| `"released"` + +The recorded fate of a child whose provider execution was RETAINED: admitted durably, with no +accepted terminal result when local observation stopped (the `RetainedExecutionPendingError` +path). Not a failure classification — the child is `down` either way. + +- `'pending'`: the reservation was reconciled at a floor, the cursor slot is open, and the + environment is kept so a resume can reconcile the paid execution. Only ever on the in-memory + `Settled` and the first `agent.child` payload; the journal states it as the `reconciled` + record and replay never yields it. For an executor without `releaseRetained` it means only + that nothing could be released. +- `'released'`: root settlement under `retainedAtSettlement: 'release'` destroyed the + environment (executor-confirmed) before any process recovered the execution; this is the + node's terminal record. The pool's own admission fault, if the reconcile raised one, is not + on this record. + +Absent = an ordinary child. The live `Settled` a driver branched on carried `'pending'` where +replay yields `'released'` for the same seq, so a resume-aware driver must not branch on the +two values. No `'recovered'` value exists yet: a live-adopted recovery settles on the ordinary +path and the recorded-result path writes no marker. + +*** + ### SupervisedResult -> **SupervisedResult**\<`Out`\> = \{ `kind`: `"winner"`; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \} \| `object` & \{ `reason`: `"all-children-down"` \| `"budget-exhausted"` \| `"aborted"`; \} \| \{ `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `leakedReservations?`: `ReadonlyArray`\<[`LeakedReservation`](#leakedreservation)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `error`: [`NoWinnerError`](#nowinnererror); \} +> **SupervisedResult**\<`Out`\> = \{ `kind`: `"winner"`; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `fleetYield`: [`FleetYield`](#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \} \| `object` & \{ `reason`: `"all-children-down"` \| `"budget-exhausted"` \| `"aborted"`; \} \| \{ `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `leakedReservations?`: `ReadonlyArray`\<[`LeakedReservation`](#leakedreservation)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `fleetYield`: [`FleetYield`](#fleetyield); `error`: [`NoWinnerError`](#nowinnererror); \} Typed terminal result (M2) — a no-winner is NEVER coerced to a best-effort output. @@ -29633,7 +29776,7 @@ Typed terminal result (M2) — a no-winner is NEVER coerced to a best-effort out ##### Type Literal -\{ `kind`: `"winner"`; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \} +\{ `kind`: `"winner"`; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `fleetYield`: [`FleetYield`](#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \} ###### kind @@ -29701,6 +29844,12 @@ Settled children whose executor teardown was never acknowledged — the resource The journaled nodes whose usage accounting is incomplete — the named gaps behind a `false` `tokensKnown`/`usdKnown` on `spentTotal`. Present exactly when non-empty. +###### fleetYield + +> **fleetYield**: [`FleetYield`](#fleetyield) + +How the fleet ended, see `FleetYield`. Always present: zeros are facts. + ###### spentBreakdown? > `optional` **spentBreakdown?**: `object` @@ -29727,7 +29876,7 @@ Where `spentTotal` went: `driverInference` = the drivers' own chat turns (metere ##### Type Literal -\{ `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `leakedReservations?`: `ReadonlyArray`\<[`LeakedReservation`](#leakedreservation)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `error`: [`NoWinnerError`](#nowinnererror); \} +\{ `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootProviderModel?`: [`RootProviderModelEvidence`](#rootprovidermodelevidence); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: `ReadonlyArray`\<[`UnconfirmedTeardown`](#unconfirmedteardown)\>; `leakedReservations?`: `ReadonlyArray`\<[`LeakedReservation`](#leakedreservation)\>; `spendGaps?`: `ReadonlyArray`\<[`SpendGap`](#spendgap)\>; `fleetYield`: [`FleetYield`](#fleetyield); `error`: [`NoWinnerError`](#nowinnererror); \} ###### kind @@ -29804,6 +29953,12 @@ Budget reservations still open when the run reached its join barrier, each named The journaled nodes whose usage accounting is incomplete — the named gaps behind a `false` `tokensKnown`/`usdKnown` on `spentTotal`. Present exactly when non-empty. +###### fleetYield + +> **fleetYield**: [`FleetYield`](#fleetyield) + +How the fleet ended, see `FleetYield`. Always present: zeros are facts. + ###### error > **error**: [`NoWinnerError`](#nowinnererror) @@ -36309,7 +36464,7 @@ a stamp asserting something that never happened. ### supervise() -> **supervise**(`profile`, `task`, `opts`): `Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error`: [`NoWinnerError`](#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \}\> +> **supervise**(`profile`, `task`, `opts`): `Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error`: [`NoWinnerError`](#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \}\> **`Stable`** @@ -36331,7 +36486,7 @@ One-call supervisor: build + run a supervisor from its exact profile. #### Returns -`Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `error`: [`NoWinnerError`](#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \}\> +`Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `error`: [`NoWinnerError`](#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](#treeview); `spentTotal`: [`Spend`](#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](#spendgap)[]; `fleetYield`: [`FleetYield`](#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](#spend); `childWork`: [`Spend`](#spend); \}; \}\> *** diff --git a/docs/api/testing.md b/docs/api/testing.md index 9f32e7f2..a6dabc19 100644 --- a/docs/api/testing.md +++ b/docs/api/testing.md @@ -916,10 +916,11 @@ An explicit run deadline always wins. Omit/`0` = immediate teardown. > `readonly` `optional` **retainedAtSettlement?**: `"release"` \| `"keep"` What root settlement does with provider environments that settled children still hold for a -retained execution: `'release'` them with one `environment-teardown` receipt each, or `'keep'` -them for a later call that resumes this run. Default: `'keep'` with `runDir` (re-running the -same `runDir` and `runId` resumes), `'release'` without it (nothing can resume an in-memory -run). See `SupervisorOpts.retainedAtSettlement`. +retained execution: `'release'` them with one `environment-teardown` receipt each and close +each released child's cursor slot with a terminal record marked `retainedExecution: +'released'`, or `'keep'` them for a later call that resumes this run. Default: `'keep'` with +`runDir` (re-running the same `runDir` and `runId` resumes), `'release'` without it (nothing +can resume an in-memory run). See `SupervisorOpts.retainedAtSettlement`. ###### Inherited from @@ -1758,10 +1759,11 @@ An explicit run deadline always wins. Omit/`0` = immediate teardown. > `readonly` `optional` **retainedAtSettlement?**: `"release"` \| `"keep"` What root settlement does with provider environments that settled children still hold for a -retained execution: `'release'` them with one `environment-teardown` receipt each, or `'keep'` -them for a later call that resumes this run. Default: `'keep'` with `runDir` (re-running the -same `runDir` and `runId` resumes), `'release'` without it (nothing can resume an in-memory -run). See `SupervisorOpts.retainedAtSettlement`. +retained execution: `'release'` them with one `environment-teardown` receipt each and close +each released child's cursor slot with a terminal record marked `retainedExecution: +'released'`, or `'keep'` them for a later call that resumes this run. Default: `'keep'` with +`runDir` (re-running the same `runDir` and `runId` resumes), `'release'` without it (nothing +can resume an in-memory run). See `SupervisorOpts.retainedAtSettlement`. ###### Inherited from @@ -2884,7 +2886,7 @@ Alias for graph tests written before `RunGraphOptions.brain` was production. The ### superviseWithTestBrain() -> **superviseWithTestBrain**(`profile`, `task`, `opts`): `Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\> +> **superviseWithTestBrain**(`profile`, `task`, `opts`): `Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\> Deterministic scripted-brain path for tests. Not exported from Runtime's main entry. @@ -2904,7 +2906,7 @@ Deterministic scripted-brain path for tests. Not exported from Runtime's main en #### Returns -`Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\> +`Promise`\<\{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"aborted"` \| `"all-children-down"` \| `"budget-exhausted"`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error?`: `undefined`; `reason`: `"cancelled"`; `source`: `string`; `cancellationReason`: `string`; `operationId?`: `string`; \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"no-winner"`; `reason`: `"driver-failed"`; `tree`: [`TreeView`](runtime.md#treeview); `downCount`: `number`; `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `leakedReservations?`: readonly [`LeakedReservation`](runtime.md#leakedreservation)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `error`: [`NoWinnerError`](runtime.md#nowinnererror); \} \| \{ `rootProviderModel`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `kind`: `"winner"`; `out`: `unknown`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `tree`: [`TreeView`](runtime.md#treeview); `spentTotal`: [`Spend`](runtime.md#spend); `rootStream?`: [`RootStreamReceipt`](durable.md#rootstreamreceipt); `providerModel?`: [`ProviderModelExecutionEvidence`](runtime.md#providermodelexecutionevidence); `teardownUnconfirmed?`: readonly [`UnconfirmedTeardown`](runtime.md#unconfirmedteardown)[]; `spendGaps?`: readonly [`SpendGap`](runtime.md#spendgap)[]; `fleetYield`: [`FleetYield`](runtime.md#fleetyield); `spentBreakdown?`: \{ `driverInference`: [`Spend`](runtime.md#spend); `childWork`: [`Spend`](runtime.md#spend); \}; \}\> *** diff --git a/src/durable/observer-projection.ts b/src/durable/observer-projection.ts index aef0feb6..9c562518 100644 --- a/src/durable/observer-projection.ts +++ b/src/durable/observer-projection.ts @@ -4,6 +4,7 @@ import type { ExecutionBindingReceipt, ProfileMaterializationReceipt, ProviderModelExecutionEvidence, + RetainedExecutionState, Spend, SpendChannel, SpendGap, @@ -185,6 +186,15 @@ export interface PursuitNodeProjection { readonly valid?: boolean readonly reason?: string readonly infra?: boolean + /** Recorded by Runtime on the `agent.child` payload: `'pending'` at a retained child's + * settlement, `'released'` when root settlement destroyed its environment without recovery. + * The status stays `down` — the split is a sibling fact, not a fourth status. A second + * `agent.child` for one node is already how a live-recovered child flips down→done, so the + * fold overwrites in observed order; that event's `settledAt` is the original settlement, so + * `settledAt` and `timing` do not move. */ + readonly retainedExecution?: RetainedExecutionState + /** When the release sweep closed a retained node's slot; absent unless `'released'`. */ + readonly releasedAt?: number /** Each channel on which the settled spend exceeded the node's reservation. The status is the * node's own outcome: a `done` node that overspent still delivered its output. */ readonly budgetViolation?: BudgetViolation @@ -268,6 +278,8 @@ type MutableNode = { valid?: boolean reason?: string infra?: boolean + retainedExecution?: RetainedExecutionState + releasedAt?: number budgetViolation?: BudgetViolation wait?: unknown firstSequence: number @@ -522,6 +534,10 @@ function projectNodeActivity(nodes: Map, record: ObserverRe if (reason) node.reason = reason const infra = booleanField(payload, 'infra') if (infra !== undefined) node.infra = infra + const retained = stringField(payload, 'retainedExecution') + if (retained === 'pending' || retained === 'released') node.retainedExecution = retained + const releasedAt = numberField(payload, 'releasedAt') + if (releasedAt !== undefined) node.releasedAt = releasedAt const budgetViolation = budgetViolationField(payload) if (budgetViolation) node.budgetViolation = budgetViolation if (payload && Object.hasOwn(payload, 'wait')) node.wait = payload.wait diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index d16fd864..29fcb96a 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -39,6 +39,7 @@ import type { ProviderModelAttemptEvidence, ProviderModelExecutionEvidence, ResultBlobStore, + RetainedExecutionState, Runtime, Settled, SpawnEvent, @@ -1142,6 +1143,8 @@ export async function replaySpawnTree( : { providerModel: copyProviderModelEvidence(ev.providerModel) }), ...budgetViolationOf(ev), trace: ev.trace ?? { status: 'unavailable', reason: 'execution-did-not-start' }, + ...(ev.harnessTranscript === undefined ? {} : { harnessTranscript: ev.harnessTranscript }), + ...(ev.retainedExecution === undefined ? {} : { retainedExecution: ev.retainedExecution }), ...settlementTime(ev.at), seq: ev.seq, }) @@ -1166,6 +1169,9 @@ export async function replaySpawnTree( // A field the journal holds and replay drops is the #1214 bug again: a reader of the // replayed tree would see no transcript where the record says there is one. ...(ev.harnessTranscript === undefined ? {} : { harnessTranscript: ev.harnessTranscript }), + // Replay yields the settlement the driver saw except for the one deliberate difference: + // live carried `'pending'` while recoverable; the journal records how the slot closed. + ...(ev.retainedExecution === undefined ? {} : { retainedExecution: ev.retainedExecution }), ...settlementTime(ev.at), seq: ev.seq, }) @@ -1309,6 +1315,7 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.outRef = ev.outRef node.trace = traceEvidenceFor(ev) node.budgetViolation = budgetViolationOf(ev).budgetViolation + if (ev.retainedExecution !== undefined) node.retainedExecution = ev.retainedExecution const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } else if (ev.kind === 'woken') { @@ -1326,6 +1333,7 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.providerModel = copyProviderModelEvidence(ev.providerModel) node.outRef = ev.outRef node.budgetViolation = budgetViolationOf(ev).budgetViolation + if (ev.retainedExecution !== undefined) node.retainedExecution = ev.retainedExecution const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } @@ -1410,6 +1418,7 @@ interface MutableSnapshot { outRef?: string trace?: NodeSnapshot['trace'] budgetViolation?: BudgetViolation + retainedExecution?: RetainedExecutionState settledAt?: number spawnedAt?: number } @@ -1481,6 +1490,7 @@ function freezeSnapshot(node: MutableSnapshot): NodeSnapshot { outRef: node.outRef, trace: node.trace, ...(node.budgetViolation === undefined ? {} : { budgetViolation: node.budgetViolation }), + ...(node.retainedExecution === undefined ? {} : { retainedExecution: node.retainedExecution }), settledAt: node.settledAt, spawnedAt: node.spawnedAt, } diff --git a/src/durable/tests/observer-projection.test.ts b/src/durable/tests/observer-projection.test.ts index 9b28e4b8..bfdefcd3 100644 --- a/src/durable/tests/observer-projection.test.ts +++ b/src/durable/tests/observer-projection.test.ts @@ -363,6 +363,68 @@ describe('projectPursuit', () => { ]) }) + it('folds a retained child from pending to released on one node without moving its settlement', () => { + // Runtime records the settlement first (`pending`, with the driver's own metering) and the + // release later (`released`, no metering). One node, two events: the marker overwrites in + // observed order, the settlement instant stays, the release instant is added, and the + // driver's inference is counted once. + const settledAt = 2 + const releasedAt = 9 + const floor = spend(7, 3, 0, { tokensKnown: false, usdKnown: false }) + const childEvent = ( + id: string, + timestamp: number, + payload: Record, + ): Parameters[0][number] => ({ + kind: 'event', + event: { + id, + pursuitId: 'pursuit:test', + runId: 'run:retained', + target: 'agent.child', + phase: 'after', + timestamp, + parentId: 'run:retained', + payload: { childId: 'root:s0', status: 'down', ...payload }, + }, + }) + const view = projectPursuit( + chain([ + spawn('run:retained', 'root:s0', 'run:retained', 'retained'), + spawn('run:retained', 'root:s1', 'run:retained', 'plain'), + childEvent('root:s0:settled', settledAt, { + retainedExecution: 'pending', + settledAt, + spent: floor, + metered: spend(5, 1, 0.001), + }), + settle('run:retained', 'root:s1', 'run:retained', spend(1, 1, 0.0001)), + childEvent('root:s0:released', releasedAt, { + retainedExecution: 'released', + releasedAt, + settledAt, + spent: floor, + }), + ]), + ) + const node = view.nodes.find((entry) => entry.id === 'root:s0') + expect(node).toMatchObject({ + status: 'down', + retainedExecution: 'released', + settledAt, + releasedAt, + eventCount: 3, + }) + expect(node?.ownInference).toMatchObject({ tokens: { input: 5, output: 1 } }) + expect(node?.spent).toMatchObject({ tokens: { input: 7, output: 3 } }) + expect(view.runs[0]?.spendGaps).toEqual([ + { id: 'root:s0', label: 'retained', kind: 'unreported', channels: ['tokens', 'usd'] }, + ]) + const plain = view.nodes.find((entry) => entry.id === 'root:s1') + expect(plain).not.toHaveProperty('retainedExecution') + expect(plain).not.toHaveProperty('releasedAt') + }) + it('keeps a reported, estimated, partly-priced and unpriced cost distinguishable', () => { const view = projectPursuit( chain([ diff --git a/src/runtime/harness-transcript.ts b/src/runtime/harness-transcript.ts index 6359e37c..e794d19f 100644 --- a/src/runtime/harness-transcript.ts +++ b/src/runtime/harness-transcript.ts @@ -190,9 +190,12 @@ interface Enumeration { /** * Enumerate candidate transcript paths inside the environment. * - * `find` is given the roots and prints one `sizepath` line per file. A missing root is not - * an error: a child that never used a harness has no directory for it. A line with no size (an - * environment whose `find` lacks `-printf`) is read and bounded after the read, as before. + * `find -printf` is given the roots and prints one `sizepath` line per file. A missing root + * is not an error: a child that never used a harness has no directory for it. GNU find is the + * target (the Tangle Linux box); BSD and BusyBox `find` have no `-printf`, their error is + * discarded by `2>/dev/null`, and the empty listing then reads as `no-transcript` — that is a + * known limit, not a fallback. A path-only line (no size) is still accepted and bounded after + * the read, which is what a stub or a wrapped `find` produces, not what BSD does. * * Two omissions used to be silent, so `skippedCount` could read 0 on an incomplete capture: a * `-size` filter dropped oversized files before they were ever listed, and `head` cut the listing diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 962cee5b..46a6127e 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -1093,6 +1093,7 @@ export type { ExecutorResult, ExecutorTeardownWarning, ExecutorToolCall, + FleetYield, Handle, MaterializedExecutionIdentity, MaterializedModelIdentity, @@ -1109,6 +1110,7 @@ export type { ResultBlobStore, ResumedKeyState, ResumedWork, + RetainedExecutionState, RootHandle, RootMaterialization, RootProviderModelEvidence, diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 5b2adc9f..8e74a1e7 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -30,7 +30,7 @@ import { type Sha256Digest, sha256DigestSchema, } from '@tangle-network/agent-interface' -import { contentAddress } from '../../durable/spawn-journal' +import { closesCursorSlot, contentAddress } from '../../durable/spawn-journal' import { ValidationError } from '../../errors' import { notifyRuntimeHookEvent, type RuntimeHooks } from '../../runtime-hooks' import { @@ -130,6 +130,7 @@ import type { ResultBlobStore, ResumedKeyState, ResumedWork, + RetainedExecutionState, Scope, Settled, SpawnEvent, @@ -357,6 +358,11 @@ interface LiveChild { spent: Spend /** The overspend its reconciliation returned. Every terminal record of this node carries it. */ budgetViolation?: BudgetViolation + /** The overspend a RETAINED reconcile returned, held back from the open-slot surfaces (the + * driver's settlement, the tree, the first `agent.child`) because the floor is not the + * execution's final spend. The release sweep restores it onto `budgetViolation` before it + * writes the terminal record, so that record agrees with what the pool committed. */ + retainedViolation?: BudgetViolation recoveryReady?: Promise acceptedResult?: ExecutorResult recoveryPending?: boolean @@ -367,6 +373,12 @@ interface LiveChild { providerModel?: import('./types').ProviderModelExecutionEvidence /** Exact terminal timestamp committed to the journal. */ settledAt?: number + /** The cursor seq `next()` stamped on this child's settlement. A retained-pending child's + * terminal record is written later by the release sweep under THIS seq, so replay yields it + * at the position the driver saw it; the sweep never mints a new `cursorSeq` for it. */ + settledSeq?: number + /** Mirrors the journal's statement about a retained execution so `makeTreeView` reports it. */ + retainedExecution?: RetainedExecutionState /** Resolves with the terminal settlement WITHOUT a `seq` — `next()` stamps the seq. */ readonly settled: Promise /** Synchronous mirror of `settled`'s value once it has resolved (else `undefined`). */ @@ -2023,6 +2035,9 @@ export function createScope(args: ScopeArgs): Scope { }, ), ) + // One tree read for the whole sweep: the duplicate guard below only needs the records that + // existed before it, and a 35-child release must not cost 35 full-tree reads. + const prior = released.length > 0 ? ((await args.journal.loadTree(args.root)) ?? []) : [] for (const { child, receipts } of released) { for (const receipt of receipts) { await appendEnvironmentTeardown( @@ -2033,6 +2048,68 @@ export function createScope(args: ScopeArgs): Scope { new Date(now()).toISOString(), ) } + // A retained-pending child whose environment is now gone can never be recovered, so its + // cursor slot closes here with the settlement the driver already received, under the seq + // `next()` stamped on it. Measured 2026-09-15: 0 of 35 reconciled children on one pursuit + // ever settled, and 185 of 223 lost sandbox children across 385 runs stopped at + // `reconciled` — every one read as `never-settled` and charged its ceiling by every + // journal reader although the pool had committed the floor. The gate is the same + // `cleanupConfirmed` the barrier reads: a `destroyed: false` receipt, an executor throw, an + // environment-less `[]` answer or a refused `confirmTeardown` leave the slot open, because + // the environment may still exist and `[].every` would otherwise close it vacuously. A + // crash between the receipt and this record leaves the slot open too; nothing below can + // run without the executor's confirmation in hand. A manager settled on the ordinary path + // already has its record and never passes `recoveryPending`. + if ( + child.cleanupConfirmed && + child.recoveryPending === true && + child.resolved?.kind === 'down' && + child.settledSeq !== undefined && + child.settledAt !== undefined && + !prior.some((event) => event.id === child.id && closesCursorSlot(event)) + ) { + child.budgetViolation = child.retainedViolation + const settledAt = child.settledAt + await args.journal.appendEvent( + args.root, + terminalDownEvent( + child, + child.resolved, + child.settledSeq, + new Date(settledAt).toISOString(), + 'released', + ), + ) + child.retainedExecution = 'released' + const releasedAt = now() + // A second `agent.child` for one node is already how a live-recovered child flips + // down→done, so the projection folds this in observed order; `settledAt` stays the + // settlement instant and `metered` is omitted so the driver's inference is not summed twice. + notifyRuntimeHookEvent( + args.hooks, + { + id: `${child.id}:released`, + runId: args.root, + target: 'agent.child', + phase: 'after', + timestamp: releasedAt, + stepIndex: child.settledSeq, + parentId: args.parentId, + payload: { + childId: child.id, + status: 'down', + retainedExecution: 'released', + releasedAt, + ...(child.resolved.outRef === undefined ? {} : { outRef: child.resolved.outRef }), + reason: child.resolved.reason, + infra: child.resolved.infra, + spent: child.spent, + ...settledNodeEvidence(child, { ...child.resolved, metered: undefined }, settledAt), + }, + }, + { signal: args.signal }, + ) + } } }) registerScopeRetainedOwner(scope as Scope, { @@ -2586,6 +2663,7 @@ async function finalizeSettlement( if (child.wait) return finalizeWait(child, settlement, seq, args, now, handle) const settledAt = now() child.settledAt = settledAt + child.settledSeq = seq const at = new Date(settledAt).toISOString() // A terminal cursor must never hide inference that has not reached the parent journal. if (settlement.metered) { @@ -2597,40 +2675,35 @@ async function finalizeSettlement( child.outRef = settlement.outRef child.trace = settlement.trace child.providerModel = settlement.providerModel - if (!child.recoveryPending) - await args.journal.appendEvent(args.root, { - ...(cancellation === undefined - ? { kind: 'settled' as const, status: 'down' as const } - : { kind: 'cancelled' as const, source: cancellation.source }), - id: child.id, - spent: child.spent, - infra: settlement.infra, - reason: settlement.reason, - ...(settlement.outRef ? { outRef: settlement.outRef } : {}), - ...(settlement.providerModel ? { providerModel: settlement.providerModel } : {}), - ...(child.budgetViolation ? { budgetViolation: child.budgetViolation } : {}), - trace: settlement.trace, - ...(settlement.harnessTranscript - ? { harnessTranscript: settlement.harnessTranscript } - : {}), - seq, - at, - }) + const retainedPending = child.recoveryPending === true + if (!retainedPending) + await args.journal.appendEvent(args.root, terminalDownEvent(child, settlement, seq, at)) // A retained-pending node keeps its cursor slot open for recovery, so it carries no terminal - // record — but its reservation WAS reconciled, and a journal that says nothing about that + // record yet — but its reservation WAS reconciled, and a journal that says nothing about that // leaves every reader (terminal accounting, a restored pool, the tree view) charging the // ceiling the pool refunded: measured 2026-09-11, the reported `childWork` carried 4M per // retained child against a metered 10 (#1190). The floor is journaled in the settlement's - // place, outside the cursor namespace, so the slot stays open and the ledgers agree. - else if (settlement.reconciled !== undefined) - await appendReconciledFloor( - args.journal, - args.root, - child.id, - settlement.reconciled, - at, - settlement.harnessTranscript, - ) + // place, outside the cursor namespace, so the slot stays open and the ledgers agree. The + // slot closes in exactly two ways: a resume recovers the execution and settles it on the + // ordinary path, or root settlement under `retainedAtSettlement: 'release'` destroys the + // environment and the release sweep (`retainedReleasers`, above) writes this settlement as + // the terminal record with `retainedExecution: 'released'`, under the seq stamped here. + else { + child.retainedExecution = 'pending' + if (settlement.reconciled !== undefined) + await appendReconciledFloor( + args.journal, + args.root, + child.id, + settlement.reconciled, + at, + settlement.harnessTranscript, + ) + } + // The in-memory down and the first `agent.child` are the only surfaces that exist while the + // slot is open, so they are where `'pending'` lives; the journal states the same fact as the + // `reconciled` record and replay never yields it. + const retainedExecution = retainedPending ? { retainedExecution: 'pending' as const } : {} notifyRuntimeHookEvent( args.hooks, { @@ -2647,6 +2720,7 @@ async function finalizeSettlement( ...(settlement.outRef === undefined ? {} : { outRef: settlement.outRef }), reason: settlement.reason, infra: settlement.infra, + ...retainedExecution, spent: child.spent, ...settledNodeEvidence(child, settlement, settledAt), }, @@ -2658,6 +2732,7 @@ async function finalizeSettlement( handle, reason: settlement.reason, infra: settlement.infra, + ...retainedExecution, ...(settlement.outRef === undefined ? {} : { outRef: settlement.outRef }), ...(settlement.providerModel ? { providerModel: settlement.providerModel } : {}), ...(child.budgetViolation ? { budgetViolation: child.budgetViolation } : {}), @@ -2733,6 +2808,38 @@ async function finalizeSettlement( * absent — an unreported receipt must not read as an empty one. Snapshots are detached because * an observer may serialize them after the live child has moved on. */ +/** The one builder of a down child's terminal record, for both its writers: the settle path at + * the reconcile and the release sweep closing a retained slot later. Two hand-written literals + * for one record shape is how a field lands on one path and not the other (#1244 was exactly + * that for `harnessTranscript`). `cancelled` keeps its `source` from the child's own + * cancellation reason, so a released cancelled child records the same source it settled with. */ +function terminalDownEvent( + child: LiveChild, + settlement: Extract, + seq: number, + at: string, + retainedExecution?: 'released', +): Extract { + const cancellation = child.cancellationReason + return { + ...(cancellation === undefined + ? { kind: 'settled' as const, status: 'down' as const } + : { kind: 'cancelled' as const, source: cancellation.source }), + id: child.id, + spent: child.spent, + infra: settlement.infra, + reason: settlement.reason, + ...(settlement.outRef ? { outRef: settlement.outRef } : {}), + ...(settlement.providerModel ? { providerModel: settlement.providerModel } : {}), + ...(child.budgetViolation ? { budgetViolation: child.budgetViolation } : {}), + trace: settlement.trace, + ...(settlement.harnessTranscript ? { harnessTranscript: settlement.harnessTranscript } : {}), + ...(retainedExecution === undefined ? {} : { retainedExecution }), + seq, + at, + } +} + function settledNodeEvidence( child: LiveChild, settlement: PreSeqSettled, @@ -3157,8 +3264,9 @@ async function runChild( // for exactly the children measured above. Every channel is marked unknown, because the // remote execution may still be consuming what it was handed off to: the floor is never read // back as a measurement. The child-work part is journaled as this node's `reconciled` floor - // (its slot stays open, so no `settled` record can carry it), and a driver's own inference - // rides its `metered` record as on every other path. + // (its slot stays open until a resume recovers it or the release sweep closes it, so no + // `settled` record carries it yet), and a driver's own inference rides its `metered` + // record as on every other path. // // A ticket already reconciled at a measured terminal spend (a persistence failure after the // artifact landed) keeps that measurement: `live.spent` is then what the pool committed. @@ -3168,9 +3276,13 @@ async function runChild( live.spent = { ...unknownFloor(accounting?.reported ?? live.spent), ms } reconcileOnce({ ...unknownFloor(accounting?.reservation ?? live.spent), ms }) } - // The node keeps its cursor slot open, so no terminal record can carry an overspend, and - // the floor is not the execution's final spend. The recovered settlement reports it from - // the recorded result; reporting it here would make the live views disagree with replay. + // The node keeps its cursor slot open, so no terminal record can carry an overspend yet, and + // the floor is not the execution's final spend. A recovered settlement reports it from the + // recorded result; reporting it here would make the live views disagree with replay. The + // pool did commit it, though, and when the run RELEASES the node the floor is its final + // charge — so the answer is held aside for the release sweep's terminal record rather than + // discarded, or that record would say 'within reservation' for a node the pool overspent. + live.retainedViolation = live.budgetViolation live.budgetViolation = undefined return { ...downRecord( @@ -3326,6 +3438,7 @@ function makeTreeView(root: NodeId, children: Map): TreeView ...(c.trace ? { trace: c.trace } : {}), ...(c.providerModel ? { providerModel: c.providerModel } : {}), ...(c.budgetViolation ? { budgetViolation: c.budgetViolation } : {}), + ...(c.retainedExecution ? { retainedExecution: c.retainedExecution } : {}), })) return { root, diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 93051ab2..11353a45 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -1663,10 +1663,11 @@ export interface SuperviseOptions { readonly childSettleGraceMs?: number | null /** * What root settlement does with provider environments that settled children still hold for a - * retained execution: `'release'` them with one `environment-teardown` receipt each, or `'keep'` - * them for a later call that resumes this run. Default: `'keep'` with `runDir` (re-running the - * same `runDir` and `runId` resumes), `'release'` without it (nothing can resume an in-memory - * run). See `SupervisorOpts.retainedAtSettlement`. + * retained execution: `'release'` them with one `environment-teardown` receipt each and close + * each released child's cursor slot with a terminal record marked `retainedExecution: + * 'released'`, or `'keep'` them for a later call that resumes this run. Default: `'keep'` with + * `runDir` (re-running the same `runDir` and `runId` resumes), `'release'` without it (nothing + * can resume an in-memory run). See `SupervisorOpts.retainedAtSettlement`. */ readonly retainedAtSettlement?: 'release' | 'keep' /** Resolve one custom external-harness session per trusted manager identity. Use this instead of diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index 62e62cc5..57926590 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -40,6 +40,7 @@ import { closesCursorSlot, contentAddress, loadSpawnForest, + type SpawnForest, } from '../../durable/spawn-journal' import { RuntimeRunStateError, ValidationError } from '../../errors' import { addSpend } from '../util' @@ -72,6 +73,7 @@ import { detachedSnapshot } from './snapshot' import type { Agent, ExecutionBindingReceipt, + FleetYield, NodeId, NoWinnerError, ProfileMaterializationReceipt, @@ -786,7 +788,7 @@ export function createSupervisor(): Supervisor { // ONE ledger: the journal. `settled` events carry spawned-child WORK; `metered` events carry // the drivers' OWN inference (the twin of `pool.observe`). `spentTotal` is their sum and the // breakdown keeps the two separable — the A++ view of where the tokens went. No pool bridge. - const { spentTotal, childWork, driverInference, gaps, providerModel } = + const { spentTotal, childWork, driverInference, gaps, providerModel, fleetYield } = await terminalAccounting(journal, opts.runId, now() - runEpochMs) return { kind: 'winner', @@ -797,6 +799,7 @@ export function createSupervisor(): Supervisor { providerModel, ...(teardownUnconfirmed.length > 0 ? { teardownUnconfirmed } : {}), ...(gaps.length > 0 ? { spendGaps: gaps } : {}), + fleetYield, ...(isNonEmptySpend(driverInference) ? { spentBreakdown: { driverInference, childWork } } : {}), @@ -815,7 +818,7 @@ export function createSupervisor(): Supervisor { // A no-winner still incurred real conserved spend before failing, so it carries `spentTotal` // summed off the SAME journal the winner path reads — the caller always learns the cost. async function noWinner(rejection?: DriverRejection): Promise> { - const { spentTotal, gaps, providerModel } = await terminalAccounting( + const { spentTotal, gaps, providerModel, fleetYield } = await terminalAccounting( journal, opts.runId, now() - runEpochMs, @@ -829,6 +832,7 @@ export function createSupervisor(): Supervisor { ...(teardownUnconfirmed.length > 0 ? { teardownUnconfirmed } : {}), ...(leakedReservations.length > 0 ? { leakedReservations } : {}), ...(gaps.length > 0 ? { spendGaps: gaps } : {}), + fleetYield, } // The lifecycle causes outrank the driver's own rejection, so they are asked first and a // proven one ends it. `undefined` means the supervisor's own state explains nothing. @@ -1034,13 +1038,23 @@ function createIntensityBreaker(opts: SupervisorOpts, trip: () => void): Intensi /** Decorate the journal so the breaker observes every `settled`-`down` event the scope * appends, without the supervisor intercepting `scope.next()`. The decorator is - * transparent — it forwards every method verbatim and only reads the down events. */ + * transparent — it forwards every method verbatim and only reads the down events. + * + * A record marked `retainedExecution: 'released'` is skipped: it is the driver's earlier down + * re-stated by the release sweep, which runs AFTER the join barrier and before the result is + * classified. Counting it would trip an armed breaker on a run that has already settled and turn + * a delivered winner into `all-children-down`, and its backdated `at` (the settlement instant) + * would sit out of order in the sliding window. A retained child was never counted at its + * settlement either (it writes no `settled` record then), so `downCount` stays what it was: + * ordinary downs only; `fleetYield.down` is where a retained child is counted. */ function wrapJournalForBreaker(journal: SpawnJournal, breaker: IntensityBreaker): SpawnJournal { return { loadTree: (root) => journal.loadTree(root), beginTree: (root, at) => journal.beginTree(root, at), appendEvent: (root, ev: SpawnEvent) => { - if (ev.kind === 'settled' && ev.status === 'down') breaker.recordDown(Date.parse(ev.at)) + if (ev.kind === 'settled' && ev.status === 'down' && ev.retainedExecution === undefined) { + breaker.recordDown(Date.parse(ev.at)) + } return journal.appendEvent(root, ev) }, } @@ -1235,6 +1249,7 @@ async function terminalAccounting( driverInference: Spend gaps: SpendGap[] providerModel: import('./types').ProviderModelExecutionEvidence + fleetYield: FleetYield }> { const events = await journal.loadTree(root) if (events === undefined) { @@ -1246,6 +1261,7 @@ async function terminalAccounting( const gaps = spendGapsFromEvents(events) const forest = await loadSpawnForest(journal, root) const providerModel = aggregateProviderModelEvidence(forest) + const fleetYield = fleetYieldFromForest(forest) const summed = addSpend(childWork, driverInference) const spentTotal: Spend = { ...summed, @@ -1254,7 +1270,58 @@ async function terminalAccounting( summed.tokensKnown !== false && !gaps.some((gap) => gap.channels.includes('tokens')), usdKnown: summed.usdKnown !== false && !gaps.some((gap) => gap.channels.includes('usd')), } - return { spentTotal, childWork, driverInference, gaps, providerModel } + return { spentTotal, childWork, driverInference, gaps, providerModel, fleetYield } +} + +/** How the run's spawned children ended, by node id across every tree of the forest. Read after + * the join barrier and the release sweep, so the terminal records the sweep wrote are in it. + * Spawned records without a parent are exempt: the run root, and the re-rooted owner copy a + * nested driver writes as its tree's first record (`ownedTreeRootSpawn` strips the parent), so + * a manager is counted once, in its parent's tree. Buckets are id sets rather than record counts + * because the journal guards seq uniqueness, not one-terminal-per-node, and the invariant + * `spawned === done + down + cancelled + neverSettled` must not rest on scope discipline alone; + * it is asserted here so a future root-id terminal record cannot silently break the tally. */ +function fleetYieldFromForest(forest: SpawnForest): FleetYield { + const spawned = new Set() + // Each id lands in exactly one bucket, by its last terminal record in journal order. + const terminal = new Map() + const released = new Set() + for (const { event } of forest.events) { + if (event.kind === 'spawned') { + if (event.parent !== undefined) spawned.add(event.id) + } else if (event.kind === 'settled' || event.kind === 'cancelled') { + const bucket = event.kind === 'cancelled' ? 'cancelled' : event.status + terminal.set(event.id, bucket) + // `released` is a subset of down + cancelled BY CONSTRUCTION here, not by trust in the + // writer: only the release sweep writes the marker and it never writes `done`, but a + // hand-built `done` record carrying it must not count twice. + if (event.retainedExecution === 'released' && bucket !== 'done') released.add(event.id) + else released.delete(event.id) + } + } + const count = (bucket: 'done' | 'down' | 'cancelled'): number => + [...terminal.values()].filter((value) => value === bucket).length + const fleetYield: FleetYield = { + spawned: spawned.size, + done: count('done'), + down: count('down'), + cancelled: count('cancelled'), + neverSettled: forest.inDoubt.length, + releasedUnrecovered: released.size, + } + const accounted = + fleetYield.done + fleetYield.down + fleetYield.cancelled + fleetYield.neverSettled + if (accounted !== fleetYield.spawned) { + throw new RuntimeRunStateError( + `supervisor: fleet yield does not partition the spawned children of '${forest.root}' (${JSON.stringify(fleetYield)})`, + ) + } + if (fleetYield.releasedUnrecovered > fleetYield.down + fleetYield.cancelled) { + throw new RuntimeRunStateError( + `supervisor: released children exceed down + cancelled for '${forest.root}' (${JSON.stringify(fleetYield)})`, + ) + } + return fleetYield } /** The journaled nodes whose usage accounting is incomplete — one `SpendGap` per node+kind, diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index ccd83832..8ca72110 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -996,6 +996,13 @@ export type Settled = providerModel?: ProviderModelExecutionEvidence /** Present when the spend reconciled for this child exceeded its reservation. */ budgetViolation?: BudgetViolation + /** Present only when this child's provider execution was RETAINED (see + * `RetainedExecutionState`); absent on an ordinary down. `'pending'` on the settlement the + * driver receives at the reconcile; `'released'` on the replayed settlement of a node the + * release sweep closed. The driver and every replay reader see the fact the journal + * states, so a reader never splits this population on `reason` text — which is identical + * on every one of these children. */ + retainedExecution?: RetainedExecutionState /** Epoch ms parsed from the durable settlement/cancellation record when available. */ settledAt?: number seq: number @@ -1236,6 +1243,11 @@ export interface NodeSnapshot { readonly trace?: WorkerTraceEvidence /** Present once a settled node's measured spend exceeded its reservation. */ readonly budgetViolation?: BudgetViolation + /** Present on a retained child: `'pending'` while its cursor slot is open, `'released'` once + * the release sweep closed it. The live view (`makeTreeView`) and the journal view + * (`materializeTreeView`) state the same fact, so a settle record's `tree` answers the + * retained-vs-down question without the observer journal. */ + readonly retainedExecution?: RetainedExecutionState } /** The live tree — what `scope.view` / `RootHandle.view()` materialize for a viewer. */ @@ -1347,6 +1359,17 @@ export type SpawnEvent = harnessTranscript?: HarnessTranscriptEvidence /** Present when the reconciled spend exceeded the reservation, on either status. */ budgetViolation?: BudgetViolation + /** Written only by the release sweep, on the same tree, after every `environment-teardown` + * receipt for the node reads `destroyed: true` and the executor's own teardown confirmed. + * `spent` is this node's child-work component of the reconcile the pool committed — for + * a leaf the streamed floor itself, for a recursive executor its `accounting().reported` + * split with the remainder on its `metered` records — never the reservation ceiling. + * `reason`/`infra`/`trace`/`harnessTranscript`/`outRef`/`providerModel` are the + * settlement the driver received, verbatim; `seq` is the cursor seq stamped on that + * delivery, so replay yields it at the position the driver saw it; `at` is the settlement + * instant, and the release instant is on the receipt immediately before it. Typed so a + * `'pending'` can never be journaled: the journal states that as `reconciled`. */ + retainedExecution?: Extract seq: number at: string } @@ -1364,6 +1387,9 @@ export type SpawnEvent = harnessTranscript?: HarnessTranscriptEvidence outRef?: string budgetViolation?: BudgetViolation + /** As on `settled`: a retained child that was cancelled settles `cancelled`, and the one + * builder writes whichever kind the settlement had. */ + retainedExecution?: Extract seq: number at: string } @@ -1475,7 +1501,9 @@ export type SpawnEvent = * observed, while its cursor slot stays OPEN so a resume can recover the execution. It stands * in for the `settled` record an open node cannot carry: cost readers and a restored pool * charge this floor for the node instead of its declared ceiling, and a later `settled` or - * `cancelled` record for the same node supersedes it. A driver's own inference travels on its + * `cancelled` record for the same node supersedes it. That record has two writers: a + * resumed process's recovered settlement, or the release sweep's terminal record marked + * `retainedExecution: 'released'`. A driver's own inference travels on its * `metered` record as on every other path, so `reconciled + metered` is what the pool * committed. Its `seq` lives outside the cursor-uniqueness namespace. */ kind: 'reconciled' @@ -1510,7 +1538,11 @@ export type SpawnEvent = * the environment its executor kept for recovery could never be recovered; this is the * receipt of the supervisor's release, one per environment, naming the provider's own id * so a fleet listing can be reconciled against it. A `destroyed: false` receipt carries why - * in `detail`, and the node is then also journaled as `teardown-unconfirmed`. + * in `detail`, and the node is then also journaled as `teardown-unconfirmed`. When every + * receipt for a node is `destroyed: true` and the executor confirms teardown, the node's + * terminal `settled`/`cancelled` record with `retainedExecution: 'released'` follows on + * the same tree and closes the cursor slot; a `destroyed: false` receipt or an unconfirmed + * teardown leaves the slot open because the environment may still exist. * Informational: replay, `materializeTreeView`, and cost readers skip it, and its `seq` is * per node, outside the cursor-uniqueness namespace. */ kind: 'environment-teardown' @@ -1670,7 +1702,11 @@ export interface SupervisorOpts { * - `'release'`: the run will not be resumed, so nothing would ever reconcile or release those * environments. The join barrier releases each one and journals an `environment-teardown` * receipt per environment. Measured 2026-09-11: four settled runs held 18 of a 60-slot - * Sandbox fleet for 19 to 37 hours without it. + * Sandbox fleet for 19 to 37 hours without it. It then closes each released child's cursor + * slot with a terminal record marked `retainedExecution: 'released'`, so `spendGaps` names + * it `unreported` (a floor) rather than `never-settled` (a ceiling) and + * `fleetYield.releasedUnrecovered` counts it; a refused release leaves the slot open and + * the node in `teardownUnconfirmed`. * - `'keep'`: a later process may resume this run, so the environments stay for its recovery. * * Default: `'keep'` when `resume` is true (a durable run a later process may continue), else @@ -1786,6 +1822,55 @@ export interface SpendGap { readonly channels: ReadonlyArray } +/** + * The recorded fate of a child whose provider execution was RETAINED: admitted durably, with no + * accepted terminal result when local observation stopped (the `RetainedExecutionPendingError` + * path). Not a failure classification — the child is `down` either way. + * + * - `'pending'`: the reservation was reconciled at a floor, the cursor slot is open, and the + * environment is kept so a resume can reconcile the paid execution. Only ever on the in-memory + * `Settled` and the first `agent.child` payload; the journal states it as the `reconciled` + * record and replay never yields it. For an executor without `releaseRetained` it means only + * that nothing could be released. + * - `'released'`: root settlement under `retainedAtSettlement: 'release'` destroyed the + * environment (executor-confirmed) before any process recovered the execution; this is the + * node's terminal record. The pool's own admission fault, if the reconcile raised one, is not + * on this record. + * + * Absent = an ordinary child. The live `Settled` a driver branched on carried `'pending'` where + * replay yields `'released'` for the same seq, so a resume-aware driver must not branch on the + * two values. No `'recovered'` value exists yet: a live-adopted recovery settles on the ordinary + * path and the recorded-result path writes no marker. + */ +export type RetainedExecutionState = 'pending' | 'released' + +/** + * How this run's spawned CHILDREN ended, counted by node id off the complete journal FOREST at + * root settlement — after the join barrier and the release sweep, so the terminal records the + * sweep wrote are included, and forest-wide so a director's grandchildren count (the population + * the observer projection shows). Invariant: `spawned === done + down + cancelled + neverSettled`. + * Distinct from a no-winner's `downCount`, the breaker's tally of ordinary down settlements: a + * released record is the driver's earlier down re-stated, so the breaker skips it and + * `downCount` never includes a retained child, while `down` here does. `spendGaps` stays + * root-tree scoped, so on a nested run the two disagree by design. + */ +export interface FleetYield { + /** Every `spawned` record with a parent; the run root and an owned tree's re-rooted owner copy + * are exempt. */ + readonly spawned: number + readonly done: number + /** `settled` records with `status: 'down'`, released records included. */ + readonly down: number + readonly cancelled: number + /** Spawned with no terminal record: a crash-orphaned child, a refused release, or a retained + * executor with nothing to release. Named after `SpendGap`'s `never-settled`. */ + readonly neverSettled: number + /** Terminal records marked `retainedExecution: 'released'` — a subset of `down + cancelled`. + * A nested manager whose OWN retained execution was released counts here beside the + * grandchildren it released, because its execution was destroyed unrecovered too. */ + readonly releasedUnrecovered: number +} + /** * One channel on which a settled reservation's measured spend exceeded what it reserved. * `tokens` is in the pool's charged unit (`chargedTokens`), `usd` is measured dollars, and a @@ -1841,6 +1926,8 @@ export type SupervisedResult = /** The journaled nodes whose usage accounting is incomplete — the named gaps behind a * `false` `tokensKnown`/`usdKnown` on `spentTotal`. Present exactly when non-empty. */ spendGaps?: ReadonlyArray + /** How the fleet ended, see `FleetYield`. Always present: zeros are facts. */ + fleetYield: FleetYield /** Where `spentTotal` went: `driverInference` = the drivers' own chat turns (metered via * `Scope.meter`); `childWork` = every spawned child's reconciled spend (the journal sum). * `driverInference + childWork === spentTotal` on `iterations`/`tokens`/`usd`; the @@ -1884,6 +1971,8 @@ export type SupervisedResult = /** The journaled nodes whose usage accounting is incomplete — the named gaps behind a * `false` `tokensKnown`/`usdKnown` on `spentTotal`. Present exactly when non-empty. */ spendGaps?: ReadonlyArray + /** How the fleet ended, see `FleetYield`. Always present: zeros are facts. */ + fleetYield: FleetYield /** Never present on a lifecycle arm — the discriminant, not prose, is what makes * `if (r.reason === 'driver-failed') r.error.message` compile and every other arm refuse it. */ error?: never @@ -1933,6 +2022,8 @@ export type SupervisedResult = /** The journaled nodes whose usage accounting is incomplete — the named gaps behind a * `false` `tokensKnown`/`usdKnown` on `spentTotal`. Present exactly when non-empty. */ spendGaps?: ReadonlyArray + /** How the fleet ended, see `FleetYield`. Always present: zeros are facts. */ + fleetYield: FleetYield /** The driver's own rejection, carried across the typed no-winner boundary so the failure is * recoverable by the caller. A non-`Error` rejection is normalized, never dropped. */ error: NoWinnerError diff --git a/src/runtime/waterfall.ts b/src/runtime/waterfall.ts index 096f1701..23bd0bb2 100644 --- a/src/runtime/waterfall.ts +++ b/src/runtime/waterfall.ts @@ -42,6 +42,7 @@ interface SpawnPayload { interface SettlePayload { childId?: string status?: string + retainedExecution?: string score?: number spent?: { usd?: number; tokens?: { input?: number; output?: number } } } @@ -81,6 +82,9 @@ export function createWaterfallCollector(): WaterfallCollector { if (!id) return const span = spans.get(id) if (!span) return + // The release event's timestamp is the release instant, not the settlement; without this + // guard a released child's bar would stretch to root settlement. + if (span.endMs !== undefined && p.retainedExecution === 'released') return span.endMs = event.timestamp span.status = p.status === 'down' ? 'down' : 'done' span.usd = p.spent?.usd ?? 0 diff --git a/tests/integration/supervise-pursuit-record.test.ts b/tests/integration/supervise-pursuit-record.test.ts index effbbfb9..df9eda32 100644 --- a/tests/integration/supervise-pursuit-record.test.ts +++ b/tests/integration/supervise-pursuit-record.test.ts @@ -26,7 +26,7 @@ import { settleRecordDigest, settleRecordJson, } from '../../src/durable/settle-record' -import { FileSpawnJournal } from '../../src/durable/spawn-journal' +import { closesCursorSlot, FileSpawnJournal } from '../../src/durable/spawn-journal' import { SupervisePursuitError, supervisePursuit } from '../../src/durable/supervise-pursuit' import { providerAsExecutor } from '../../src/runtime/environment-provider' import { cancelRun, readRunCancellation } from '../../src/runtime/supervise/run-layout' @@ -343,8 +343,38 @@ describe('supervisePursuit durable terminal records', () => { }, ]) expect(events.some((event) => event.kind === 'teardown-unconfirmed')).toBe(false) - // The settle record still lands, so the run stays final. - expect(await readSettleRecord(runDir)).toBeDefined() + // The release closed the slot: one terminal record after the receipt, marked released. + const terminal = events.filter( + (event) => event.id === admitted[0]?.id && closesCursorSlot(event), + ) + expect(terminal).toMatchObject([ + { kind: 'settled', status: 'down', retainedExecution: 'released' }, + ]) + expect(events.indexOf(terminal[0]!)).toBeGreaterThan( + events.findIndex((event) => event.kind === 'environment-teardown'), + ) + // The settle record still lands, so the run stays final — and it carries the yield, the + // released marker on the tree, and the gap as a floor rather than a ceiling. + const result = await readSettleRecord(runDir) + expect(result).toBeDefined() + if (result === undefined) return + expect(result.fleetYield).toMatchObject({ releasedUnrecovered: 1, neverSettled: 0 }) + expect(result.tree.nodes.find((node) => node.id === admitted[0]?.id)).toMatchObject({ + retainedExecution: 'released', + }) + expect(result.spendGaps).toEqual([ + expect.objectContaining({ id: admitted[0]?.id, kind: 'unreported' }), + ]) + expect(result.spendGaps?.some((gap) => gap.kind === 'never-settled')).toBe(false) + // The operator's projection says the same thing off the observer journal. + const projected = projectPursuit( + await new FileObserverJournal(settled.observerPath, 'pursuit:record').read(), + ) + expect(projected.nodes.find((node) => node.id === admitted[0]?.id)).toMatchObject({ + status: 'down', + retainedExecution: 'released', + releasedAt: expect.any(Number), + }) }) it("refuses retainedAtSettlement 'keep', which a settle record would contradict", async () => { diff --git a/tests/kernel/delegate.test.ts b/tests/kernel/delegate.test.ts index 20fdab4c..bc700fdf 100644 --- a/tests/kernel/delegate.test.ts +++ b/tests/kernel/delegate.test.ts @@ -19,7 +19,12 @@ vi.mock('../../src/runtime/supervise/supervise', () => ({ import { defaultDelegateBudget, delegate } from '../../src/runtime/supervise/delegate' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import type { Spend, SupervisedResult, TreeView } from '../../src/runtime/supervise/types' +import type { + FleetYield, + Spend, + SupervisedResult, + TreeView, +} from '../../src/runtime/supervise/types' import { testAgentProfile } from './test-agent-profile' const router: RouterTransportConfig = { @@ -46,8 +51,17 @@ const spentTotal: Spend = { ms: 5100, } +const fleetYield: FleetYield = { + spawned: 0, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 0, +} + function winner(out: unknown): SupervisedResult { - return { kind: 'winner', out, outRef: 'blob:1', tree: emptyTree, spentTotal } + return { kind: 'winner', out, outRef: 'blob:1', tree: emptyTree, spentTotal, fleetYield } } beforeEach(() => { diff --git a/tests/kernel/overspend-settlement.test.ts b/tests/kernel/overspend-settlement.test.ts index edd9c19c..a27cf68c 100644 --- a/tests/kernel/overspend-settlement.test.ts +++ b/tests/kernel/overspend-settlement.test.ts @@ -484,7 +484,7 @@ describe('an overspent child that did not complete, or whose accounting is at fa const settled = await scope.next() // The node stays open for recovery, so it has no terminal record. The recovered settlement // reports the overspend from the recorded result; the live views must not report one early. - expect(settled).toMatchObject({ kind: 'down', infra: true }) + expect(settled).toMatchObject({ kind: 'down', infra: true, retainedExecution: 'pending' }) expect(settled).not.toHaveProperty('budgetViolation') const events = (await journal.loadTree('run')) ?? [] expect(events.some((event) => event.kind === 'settled' || event.kind === 'cancelled')).toBe( @@ -493,9 +493,9 @@ describe('an overspent child that did not complete, or whose accounting is at fa expect(events.find((event) => event.kind === 'reconciled')).toMatchObject({ id: spawned.handle.id, }) - expect(scope.view.nodes.find((node) => node.id === spawned.handle.id)).not.toHaveProperty( - 'budgetViolation', - ) + const node = scope.view.nodes.find((entry) => entry.id === spawned.handle.id) + expect(node).not.toHaveProperty('budgetViolation') + expect(node).toMatchObject({ retainedExecution: 'pending' }) expect(pool.readout().reservedTokens).toBe(0) }) diff --git a/tests/kernel/retained-environment-release.test.ts b/tests/kernel/retained-environment-release.test.ts index 90b1031d..defe0c27 100644 --- a/tests/kernel/retained-environment-release.test.ts +++ b/tests/kernel/retained-environment-release.test.ts @@ -13,6 +13,14 @@ * and journals one receipt per environment, naming the provider's id. A durable run keeps them by * default, so an interrupted run is still recovered by its resume. A release the provider refuses * is receipted as a typed failure, and the node stays named as unconfirmed. + * + * A released environment can never be recovered, so the release also CLOSES the node's cursor + * slot: the settlement the driver received at the reconcile is written as the terminal record, + * marked `retainedExecution: 'released'`, under the seq the driver saw. Measured 2026-09-15: 0 of + * 35 reconciled children on one pursuit ever settled, and 185 of 223 lost sandbox children across + * 385 runs stopped at `reconciled` — read as `never-settled` and charged the ceiling although the + * pool had committed the floor. A refused release writes nothing, because the environment may + * still exist. */ import { existsSync, readFileSync } from 'node:fs' @@ -25,9 +33,14 @@ import type { AgentEnvironmentProvider, } from '@tangle-network/agent-interface/environment-provider' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { closesCursorSlot } from '../../src/durable/spawn-journal' +import { + closesCursorSlot, + materializeTreeView, + replaySpawnTree, +} from '../../src/durable/spawn-journal' import { providerAsExecutor } from '../../src/runtime/environment-provider' import { driverChild } from '../../src/runtime/supervise/driver-executor' +import { RetainedExecutionPendingError } from '../../src/runtime/supervise/retained-executor' import { createFileRunContext, createInMemoryRunContext, @@ -35,11 +48,14 @@ import { import { createSupervisor } from '../../src/runtime/supervise/supervisor' import type { Agent, + Executor, ExecutorFactory, Scope, + Settled, SpawnEvent, SupervisorOpts, } from '../../src/runtime/supervise/types' +import type { RuntimeHookEvent } from '../../src/runtime-hooks' import { durableRetainedProvider } from '../helpers/durable-retained-provider' import { testAgentProfile } from './test-agent-profile' @@ -126,6 +142,18 @@ async function spawnAndAwait(scope: Scope, worker: Agent) => events.flatMap((event) => (event.kind === 'environment-teardown' ? [event] : [])) +/** The records that close a node's cursor slot — exactly one for a released node, none while open. */ +const terminalRecords = (events: ReadonlyArray, id: string) => + events.filter((event) => event.id === id && closesCursorSlot(event)) + +const childPayloads = (hookEvents: ReadonlyArray, childId: string) => + hookEvents.flatMap((event) => + event.target === 'agent.child' && + (event.payload as { childId?: string } | undefined)?.childId === childId + ? [{ stepIndex: event.stepIndex, payload: event.payload as Record }] + : [], + ) + const admittedEnvironmentId = (events: ReadonlyArray, id: string) => events.flatMap((event) => event.kind === 'execution-admitted' && @@ -147,6 +175,8 @@ describe('retained environments at root settlement', () => { it('releases a retained-pending child environment when a run that cannot resume settles', async () => { const fleet = retainedProvider(directory) const context = createInMemoryRunContext() + const hookEvents: RuntimeHookEvent[] = [] + let live: Settled | undefined const result = await createSupervisor().run( { name: 'root', @@ -156,6 +186,7 @@ describe('retained environments at root settlement', () => { retainedWorker(providerAsExecutor(fleet.provider())), ) expect(settled?.kind).toBe('down') + live = settled // The retained execution was admitted and its result read lost: the environment is // alive at this point, exactly as the barrier used to leave it. expect(fleet.environments()).toHaveLength(1) @@ -163,7 +194,16 @@ describe('retained environments at root settlement', () => { }, }, 'task', - { ...context, runId: 'release', budget: { maxIterations: 2, maxTokens: 20 } }, + { + ...context, + runId: 'release', + budget: { maxIterations: 2, maxTokens: 20 }, + hooks: { + onEvent: (event) => { + hookEvents.push(event) + }, + }, + }, ) expect(result.kind, JSON.stringify(result)).toBe('winner') // The provider no longer holds the environment, and the run no longer names a leak. @@ -186,14 +226,333 @@ describe('retained environments at root settlement', () => { }, ]) expect(events.some((event) => event.kind === 'teardown-unconfirmed')).toBe(false) - // The release is a resource fact, not a settlement: the node's cursor slot is still open and - // its reconciled floor still stands in the settlement's place. - expect(events.filter((event) => event.id === 'release:s0' && closesCursorSlot(event))).toEqual( - [], + + // The driver's settlement said the slot was open and recoverable, and it carried the + // transcript receipt the failure path read out of the live box. + expect(live?.kind).toBe('down') + if (live?.kind !== 'down') return + expect(live.retainedExecution).toBe('pending') + expect(live.harnessTranscript).toBeDefined() + expect(live.reason).toContain('reconciliation') + // The release closed the slot: exactly one terminal record, the settlement the driver + // received verbatim (reason, infra, trace, transcript) under the seq it saw, marked as + // released, carrying the reconciled floor and never the reservation ceiling. The + // reconciled floor still exists beneath it, superseded. + const terminal = terminalRecords(events, 'release:s0') + expect(terminal).toEqual([ + { + kind: 'settled', + status: 'down', + id: 'release:s0', + retainedExecution: 'released', + infra: true, + reason: live.reason, + spent: expect.objectContaining({ tokensKnown: false, usdKnown: false }), + ...(live.providerModel === undefined ? {} : { providerModel: live.providerModel }), + trace: live.trace, + harnessTranscript: live.harnessTranscript, + seq: 0, + at: new Date(live.settledAt!).toISOString(), + }, + ]) + expect(events.indexOf(terminal[0]!)).toBeGreaterThan( + events.indexOf(releaseReceipts(events)[0]!), ) expect(events.some((event) => event.kind === 'reconciled' && event.id === 'release:s0')).toBe( true, ) + // Every reader now agrees: the gap is a floor, not a ceiling; the total is still not a + // measurement; the yield counts the node as down and released. + expect(result.spendGaps).toEqual([ + expect.objectContaining({ + id: 'release:s0', + kind: 'unreported', + channels: expect.arrayContaining(['tokens', 'usd']), + }), + ]) + expect(result.spentTotal.tokensKnown).toBe(false) + expect(result.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 1, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 1, + }) + expect(result.tree.nodes.find((node) => node.id === 'release:s0')).toMatchObject({ + status: 'failed', + retainedExecution: 'released', + }) + expect(await replaySpawnTree(context.journal, context.blobs, 'release')).toMatchObject([ + { + kind: 'down', + retainedExecution: 'released', + settledAt: live.settledAt, + seq: 0, + harnessTranscript: live.harnessTranscript, + }, + ]) + expect( + materializeTreeView(events).nodes.find((node) => node.id === 'release:s0')?.retainedExecution, + ).toBe('released') + // Two `agent.child` events for one node: the settlement (pending, with the driver's metering + // if any) and the release (released, no metering, the release instant beside the settlement). + const payloads = childPayloads(hookEvents, 'release:s0') + expect(payloads.map((entry) => entry.stepIndex)).toEqual([0, 0]) + expect(payloads.map((entry) => entry.payload)).toMatchObject([ + { status: 'down', retainedExecution: 'pending', settledAt: live.settledAt }, + { + status: 'down', + retainedExecution: 'released', + releasedAt: expect.any(Number), + settledAt: live.settledAt, + }, + ]) + expect(payloads[1]?.payload).not.toHaveProperty('metered') + }) + + it('does not trip an armed intensity breaker with the released record', async () => { + // The released record is the driver's earlier down re-stated after the join barrier. Counting + // it would abort a run that has already settled and reclassify a delivered winner. + const fleet = retainedProvider(directory) + const context = createInMemoryRunContext() + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + await spawnAndAwait(scope, retainedWorker(providerAsExecutor(fleet.provider()))) + return 'finished' + }, + }, + 'task', + { + ...context, + runId: 'breaker', + budget: { maxIterations: 2, maxTokens: 20 }, + maxRestarts: 0, + withinMs: 60_000, + }, + ) + expect(result.kind, JSON.stringify(result)).toBe('winner') + const events = (await context.journal.loadTree('breaker')) ?? [] + expect(terminalRecords(events, 'breaker:s0')).toMatchObject([ + { kind: 'settled', status: 'down', retainedExecution: 'released' }, + ]) + expect(result.fleetYield.releasedUnrecovered).toBe(1) + }) + + it('carries the overspend the pool committed onto the released record', async () => { + // The retained reconcile returned a violation the pool committed (free tokens went negative). + // The open-slot surfaces withhold it because the floor is not the execution's final spend — + // but when the run releases the node, the floor IS its final charge, and a terminal record + // that said 'within reservation' would disagree with the ledger. + let released = false + const executor: Executor = { + runtime: 'router', + execute(): AsyncIterable<{ kind: 'tokens'; input: number; output: number }> { + return (async function* () { + yield { kind: 'tokens' as const, input: 900_000, output: 0 } + throw new RetainedExecutionPendingError(new Error('provider still running')) + })() + }, + teardown: async () => + released ? { destroyed: true } : { destroyed: false, detail: 'retained' }, + releaseRetained: async () => { + released = true + return [{ provider: 'test', environmentId: 'environment-1', destroyed: true }] + }, + resultArtifact: () => { + throw new Error('retained executor has no result') + }, + } + const overspending: Agent = Object.assign( + { name: 'overspending', act: async () => 'unused' }, + { executorSpec: { profile: testAgentProfile('overspending'), harness: null, executor } }, + ) + const context = createInMemoryRunContext() + let live: Settled | undefined + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + expect( + scope.spawn(overspending, 'task', { + label: 'overspending', + budget: { maxIterations: 4, maxTokens: 800_000 }, + }).ok, + ).toBe(true) + live = await scope.next() + // Withheld while the slot is open, exactly as before. + expect(live).toMatchObject({ kind: 'down', retainedExecution: 'pending' }) + expect(live).not.toHaveProperty('budgetViolation') + expect(scope.view.nodes[0]).not.toHaveProperty('budgetViolation') + return 'finished' + }, + }, + 'task', + { ...context, runId: 'overspent', budget: { maxIterations: 100, maxTokens: 2_000_000 } }, + ) + expect(result.kind, JSON.stringify(result)).toBe('winner') + const violation = { overspent: [{ channel: 'tokens', reserved: 800_000, spent: 900_000 }] } + const events = (await context.journal.loadTree('overspent')) ?? [] + expect(terminalRecords(events, 'overspent:s0')).toMatchObject([ + { + kind: 'settled', + status: 'down', + retainedExecution: 'released', + spent: { tokens: { input: 900_000, output: 0 }, tokensKnown: false }, + budgetViolation: violation, + }, + ]) + expect(result.tree.nodes.find((node) => node.id === 'overspent:s0')).toMatchObject({ + retainedExecution: 'released', + budgetViolation: violation, + }) + expect(await replaySpawnTree(context.journal, context.blobs, 'overspent')).toMatchObject([ + { kind: 'down', retainedExecution: 'released', budgetViolation: violation }, + ]) + }) + + it('writes nothing when the executor cannot confirm teardown after a destroyed receipt', async () => { + // A refused release in its second form: the provider destroyed the environment, but the + // executor's own teardown probe fails afterwards. The slot stays open — the run cannot prove + // the environment is gone from the executor's side — and nothing terminal is written. + const fleet = retainedProvider(directory) + const context = createInMemoryRunContext() + const factory = providerAsExecutor(fleet.provider()) + const probeFailsAfterRelease: ExecutorFactory = (spec, ctx) => { + const executor = factory(spec, ctx) + let releasedOnce = false + return { + ...executor, + teardown: async (grace) => { + if (releasedOnce) throw new Error('teardown probe failed after release') + return executor.teardown(grace) + }, + releaseRetained: async (signal) => { + const receipts = await executor.releaseRetained!(signal) + releasedOnce = true + return receipts + }, + } + } + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + await spawnAndAwait(scope, retainedWorker(probeFailsAfterRelease)) + return 'finished' + }, + }, + 'task', + { ...context, runId: 'unconfirmed', budget: { maxIterations: 2, maxTokens: 20 } }, + ) + expect(result.kind, JSON.stringify(result)).toBe('winner') + expect(fleet.environments()).toEqual([]) + expect(result.teardownUnconfirmed?.map((node) => node.id)).toEqual(['unconfirmed:s0']) + const events = (await context.journal.loadTree('unconfirmed')) ?? [] + expect(releaseReceipts(events)).toMatchObject([{ id: 'unconfirmed:s0', destroyed: true }]) + expect(terminalRecords(events, 'unconfirmed:s0')).toEqual([]) + expect(result.tree.nodes.find((node) => node.id === 'unconfirmed:s0')).toMatchObject({ + retainedExecution: 'pending', + }) + expect(result.spendGaps).toEqual([expect.objectContaining({ kind: 'never-settled' })]) + expect(result.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 1, + releasedUnrecovered: 0, + }) + }) + + it('closes the slot on an aborted run that declares release, so a later resume recovers nothing', async () => { + // The sweep has no abort guard: under an explicit `'release'` a cancelled run still releases + // and closes the slot. A resume that then arrives finds the node terminal and never attempts + // recovery against an environment that no longer exists. + const fleet = retainedProvider(directory) + const runDirectory = join(directory, 'run') + const common = { + runId: 'abort-release', + budget: { maxIterations: 2, maxTokens: 20 }, + rootIdentity: { + profileDigest: canonicalCandidateDigest({ name: 'root' }), + taskDigest: canonicalCandidateDigest('task'), + }, + } satisfies Partial + const root: Agent = { + name: 'root', + async act(_task, scope) { + if (scope.resume === undefined) { + await spawnAndAwait(scope, retainedWorker(providerAsExecutor(fleet.provider()))) + return 'finished' + } + expect(scope.view.inFlight).toBe(0) + return 'resumed' + }, + } + const abort = new AbortController() + fleet.state.observe = async (signal) => { + abort.abort(new Error('operator stopped the run')) + await new Promise((_resolve, reject) => { + const fail = () => reject(signal?.reason ?? new Error('observation aborted')) + if (signal === undefined || signal.aborted) fail() + else signal.addEventListener('abort', fail, { once: true }) + }) + } + const aborted = await createSupervisor().run(root, 'task', { + ...createFileRunContext(runDirectory), + ...common, + signal: abort.signal, + retainedAtSettlement: 'release', + }) + expect(aborted.kind).toBe('no-winner') + expect(fleet.environments()).toEqual([]) + expect(fleet.state.destroys).toBe(1) + const abortedEvents = + (await createFileRunContext(runDirectory).journal.loadTree('abort-release')) ?? [] + expect(releaseReceipts(abortedEvents)).toMatchObject([ + { id: 'abort-release:s0', destroyed: true }, + ]) + // The caller's abort is a cancellation, so the child settled `cancelled` — and the released + // record keeps that kind and its source, exactly as the driver's settlement had them. + expect(terminalRecords(abortedEvents, 'abort-release:s0')).toMatchObject([ + { kind: 'cancelled', source: 'signal', retainedExecution: 'released' }, + ]) + expect(aborted.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 0, + cancelled: 1, + neverSettled: 0, + releasedUnrecovered: 1, + }) + + fleet.state.observe = undefined + fleet.state.resultLost = false + const restarted = createFileRunContext(runDirectory) + const resumed = await createSupervisor().run(root, 'task', { + ...restarted, + ...common, + resume: true, + recoverExecutor: providerAsExecutor(fleet.provider()), + }) + expect(resumed.kind, JSON.stringify(resumed)).toBe('winner') + if (resumed.kind !== 'winner') return + expect(resumed.out).toBe('resumed') + expect(fleet.state.destroys).toBe(1) + expect( + await replaySpawnTree(restarted.journal, restarted.blobs, 'abort-release'), + ).toMatchObject([{ kind: 'down', retainedExecution: 'released' }]) + expect(resumed.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 0, + cancelled: 1, + neverSettled: 0, + releasedUnrecovered: 1, + }) }) it('keeps the environment of a durable run its caller interrupts, and the resume recovers it', async () => { @@ -247,6 +606,17 @@ describe('retained environments at root settlement', () => { (await createFileRunContext(runDirectory).journal.loadTree('interrupt')) ?? [] expect(releaseReceipts(interruptedEvents)).toEqual([]) expect(interruptedEvents.some((event) => event.kind === 'teardown-unconfirmed')).toBe(true) + // `resume: true` defaults to `keep`, so the sweep never runs: the slot is still open, and + // the yield names the node never-settled, not released. + expect(terminalRecords(interruptedEvents, 'interrupt:s0')).toEqual([]) + expect(interrupted.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 1, + releasedUnrecovered: 0, + }) // A later process resumes the run and the provider delivers the retained result. fleet.state.observe = undefined @@ -267,12 +637,20 @@ describe('retained environments at root settlement', () => { expect(resumed.teardownUnconfirmed).toBeUndefined() const resumedEvents = (await restarted.journal.loadTree('interrupt')) ?? [] expect(releaseReceipts(resumedEvents)).toEqual([]) - expect( - resumedEvents.some( - (event) => - event.kind === 'settled' && event.id === 'interrupt:s0' && event.status === 'done', - ), - ).toBe(true) + const recovered = resumedEvents.find( + (event) => event.kind === 'settled' && event.id === 'interrupt:s0' && event.status === 'done', + ) + expect(recovered).toBeDefined() + // A recovered settlement is an ordinary one: no marker, and the yield counts it as done. + expect(recovered).not.toHaveProperty('retainedExecution') + expect(resumed.fleetYield).toEqual({ + spawned: 1, + done: 1, + down: 0, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 0, + }) }) it('releases on a durable run whose caller declares the settlement final', async () => { @@ -312,6 +690,11 @@ describe('retained environments at root settlement', () => { destroyed: true, }, ]) + expect(terminalRecords(events, 'final:s0')).toMatchObject([ + { kind: 'settled', status: 'down', retainedExecution: 'released', seq: 0 }, + ]) + expect(result.fleetYield.releasedUnrecovered).toBe(1) + expect(result.fleetYield.neverSettled).toBe(0) }) it('receipts a release the provider refuses as a typed failure and keeps naming the node', async () => { @@ -350,6 +733,27 @@ describe('retained environments at root settlement', () => { expect( events.some((event) => event.kind === 'teardown-unconfirmed' && event.id === 'refused:s0'), ).toBe(true) + // The environment still exists, so the slot honestly stays open: no terminal record, the + // reconciled floor stands, the gap is a ceiling, and the node is still pending. + expect(terminalRecords(events, 'refused:s0')).toEqual([]) + expect(events.some((event) => event.kind === 'reconciled' && event.id === 'refused:s0')).toBe( + true, + ) + expect(result.spendGaps).toEqual([ + expect.objectContaining({ id: 'refused:s0', kind: 'never-settled' }), + ]) + expect(result.tree.nodes.find((node) => node.id === 'refused:s0')).toMatchObject({ + status: 'failed', + retainedExecution: 'pending', + }) + expect(result.fleetYield).toEqual({ + spawned: 1, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 1, + releasedUnrecovered: 0, + }) }) it("reaches a nested manager's retained children and receipts them in the nested tree", async () => { @@ -403,8 +807,29 @@ describe('retained environments at root settlement', () => { destroyed: true, }, ]) + // The grandchild's terminal record lands in the nested tree after its receipt; the manager + // settled normally and keeps its one unmarked record in the root tree. + const nestedTerminal = terminalRecords(nested, 'root:s0:s0') + expect(nestedTerminal).toMatchObject([ + { kind: 'settled', status: 'down', retainedExecution: 'released' }, + ]) + expect(nested.indexOf(nestedTerminal[0]!)).toBeGreaterThan( + nested.indexOf(releaseReceipts(nested)[0]!), + ) const rootEvents = (await context.journal.loadTree('root')) ?? [] expect(releaseReceipts(rootEvents)).toEqual([]) expect(rootEvents.some((event) => event.kind === 'teardown-unconfirmed')).toBe(false) + const managerTerminal = terminalRecords(rootEvents, 'root:s0') + expect(managerTerminal).toHaveLength(1) + expect(managerTerminal[0]).not.toHaveProperty('retainedExecution') + // Forest scope: the manager and its grandchild both count, the grandchild as released. + expect(result.fleetYield).toEqual({ + spawned: 2, + done: 0, + down: 2, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 1, + }) }) }) diff --git a/tests/kernel/supervise-global-concurrency.test.ts b/tests/kernel/supervise-global-concurrency.test.ts index 7c2ebc3d..3cb5c9aa 100644 --- a/tests/kernel/supervise-global-concurrency.test.ts +++ b/tests/kernel/supervise-global-concurrency.test.ts @@ -277,12 +277,26 @@ describe('supervise tree-wide worker capacity', () => { (event) => event.kind === 'spawned' && event.label === 'retained', )?.id expect(retainedId).toBeDefined() - // The cursor slot stays open — a resume may still recover the execution — so no terminal - // record exists for the node; the reconciled floor stands in its place. + // The cursor slot stays open: this executor implements no `releaseRetained`, so the root's + // release sweep has nothing to release and never selects it. No terminal record exists for the + // node; the reconciled floor stands in its place, and the run reports it as never settled. + // (tests/kernel/retained-environment-release.test.ts pins the other half: an executor WITH + // `releaseRetained` gets a released terminal record at settlement.) expect(events.filter((event) => event.id === retainedId && closesCursorSlot(event))).toEqual([]) expect(events.filter((event) => event.kind === 'reconciled')).toMatchObject([ { id: retainedId, spent: { tokens: { input: 7, output: 3 }, tokensKnown: false } }, ]) + expect(result.fleetYield).toEqual({ + spawned: 2, + done: 1, + down: 0, + cancelled: 0, + neverSettled: 1, + releasedUnrecovered: 0, + }) + expect(result.spendGaps).toEqual([ + expect.objectContaining({ id: retainedId, kind: 'never-settled' }), + ]) // Every journal reader charges that floor: terminal accounting, the ceiling list a restored // pool is charged from, and the materialized tree. const { childWork } = sumSpendFromEvents(events) @@ -352,6 +366,16 @@ describe('supervise tree-wide worker capacity', () => { seq: 2, at, }, + { + kind: 'spawned', + id: 'r:s3', + parent: 'r', + label: 'released', + budget, + runtime: 'router', + seq: 3, + at, + }, { kind: 'reconciled', id: 'r:s0', spent: floor(7, 3), seq: 0, at }, { kind: 'reconciled', id: 'r:s0', spent: floor(9, 4), seq: 1, at }, { kind: 'reconciled', id: 'r:s1', spent: floor(5, 5), seq: 0, at }, @@ -364,10 +388,25 @@ describe('supervise tree-wide worker capacity', () => { seq: 0, at, }, + // A node the release sweep closed: its floor and then the terminal record carrying the same + // floor, marked released. The floor is charged once, no ceiling, and the view says why. + { kind: 'reconciled', id: 'r:s3', spent: floor(11, 2), seq: 0, at }, + { + kind: 'settled', + id: 'r:s3', + status: 'down', + infra: true, + reason: 'retained provider execution requires reconciliation before replacement', + spent: floor(11, 2), + trace: { status: 'unavailable', reason: 'execution-did-not-start' }, + retainedExecution: 'released', + seq: 1, + at, + }, ] expect(uncertainSpawnBudgets(events)).toEqual([budget]) const { childWork } = sumSpendFromEvents(events) - expect(childWork.tokens).toMatchObject({ input: 9 + 100 + 4000, output: 4 + 50 }) + expect(childWork.tokens).toMatchObject({ input: 9 + 100 + 4000 + 11, output: 4 + 50 + 2 }) expect(childWork.tokensKnown).toBe(false) const view = materializeTreeView(events) expect(view.nodes.find((n) => n.id === 'r:s0')?.spent.tokens).toMatchObject({ @@ -378,6 +417,12 @@ describe('supervise tree-wide worker capacity', () => { input: 100, output: 50, }) + expect(view.nodes.find((n) => n.id === 'r:s3')).toMatchObject({ + status: 'failed', + retainedExecution: 'released', + spent: { tokens: { input: 11, output: 2 } }, + }) + expect(view.nodes.find((n) => n.id === 'r:s0')).not.toHaveProperty('retainedExecution') }) it('holds one cap across root → manager → sub-manager → worker execution', async () => { diff --git a/tests/kernel/supervise.test.ts b/tests/kernel/supervise.test.ts index a779ac87..d6a3b0cd 100644 --- a/tests/kernel/supervise.test.ts +++ b/tests/kernel/supervise.test.ts @@ -2834,6 +2834,14 @@ describe('supervisor: terminal accounting — wall-clock ms, explicit known flag expect(result.spentTotal.tokensKnown).toBe(true) expect(result.spentTotal.usdKnown).toBe(true) expect(result.spendGaps).toBeUndefined() + expect(result.fleetYield).toEqual({ + spawned: 2, + done: 2, + down: 0, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 0, + }) }) it('a child that died before reporting keeps tokensKnown false AND is named in spendGaps', async () => { @@ -2931,6 +2939,8 @@ describe('supervisor: terminal accounting — wall-clock ms, explicit known flag channels: ['tokens', 'usd'], }, ]) + // A crash-orphaned spawn is never-settled — distinct from a released one, which is down. + expect(result.fleetYield).toMatchObject({ spawned: 1, neverSettled: 1, releasedUnrecovered: 0 }) // Wall clock anchors to the ORIGINAL root instant recorded in the journal, not this process. expect(result.spentTotal.ms).toBe(50) }) diff --git a/tests/kernel/waterfall.test.ts b/tests/kernel/waterfall.test.ts index fd27f452..5f69a9ca 100644 --- a/tests/kernel/waterfall.test.ts +++ b/tests/kernel/waterfall.test.ts @@ -79,3 +79,32 @@ describe('createWaterfallCollector', () => { expect(w.report().spans).toHaveLength(0) }) }) + +describe('a released retained child keeps its bar at the settlement instant', () => { + it('ignores the release event timestamp once the child has settled', () => { + const w = createWaterfallCollector() + w.hooks.onEvent?.(spawn('r0', 'worker:0', 1000) as never, {}) + w.hooks.onEvent?.(settle('r0', 3000, 0.01, 10, 5, undefined, true) as never, {}) + // Root settlement releases the retained environment much later and re-states the down; + // without the guard the bar would stretch from 3000 to 90000. + w.hooks.onEvent?.( + { + id: 'r0:released', + runId: 'run-1', + target: 'agent.child', + phase: 'after', + timestamp: 90_000, + payload: { + childId: 'r0', + status: 'down', + retainedExecution: 'released', + releasedAt: 90_000, + }, + } as never, + {}, + ) + const span = w.report().spans.find((entry) => entry.id === 'r0') + expect(span?.endMs).toBe(3000) + expect(span?.status).toBe('down') + }) +}) diff --git a/tests/knowledge-improvement-job.test.ts b/tests/knowledge-improvement-job.test.ts index a2fbc3e9..a39495f5 100644 --- a/tests/knowledge-improvement-job.test.ts +++ b/tests/knowledge-improvement-job.test.ts @@ -77,6 +77,14 @@ function winner(): SupervisedResult { usd: 0.004, ms: 75, }, + fleetYield: { + spawned: 0, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 0, + }, } as unknown as SupervisedResult } diff --git a/tests/mcp/delegate.test.ts b/tests/mcp/delegate.test.ts index e7e554bb..b8941ef3 100644 --- a/tests/mcp/delegate.test.ts +++ b/tests/mcp/delegate.test.ts @@ -24,7 +24,12 @@ import { } from '../../src/mcp/tools/delegate' import type { RouterTransportConfig } from '../../src/runtime/router-client' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import type { Spend, SupervisedResult, TreeView } from '../../src/runtime/supervise/types' +import type { + FleetYield, + Spend, + SupervisedResult, + TreeView, +} from '../../src/runtime/supervise/types' const router: RouterTransportConfig = { routerBaseUrl: 'http://localhost/v1', @@ -48,12 +53,21 @@ const spentTotal: Spend = { usd: 0.0019, ms: 4200, } +const fleetYield: FleetYield = { + spawned: 0, + done: 0, + down: 0, + cancelled: 0, + neverSettled: 0, + releasedUnrecovered: 0, +} const winnerResult: SupervisedResult = { kind: 'winner', out: { patch: 'diff' }, outRef: 'blob:1', tree: emptyTree, spentTotal, + fleetYield, } beforeEach(() => {