diff --git a/CHANGELOG.md b/CHANGELOG.md index acea709f..7f2e5b69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.237.1 + +**A refused environment create fails closed instead of retrying forever.** A provider SDK throws its own error classes, so a `create` the platform refused arrived at the driver's classifier as neither a `BackendTransportError` nor an `AgentEvalError` and took the foreign-accident default: retry. `classifyDriverFailure` now reads a plain HTTP status off any thrown `Error` and applies the split the transport branch already promises — 408, 429 and 5xx are the upstream having a bad moment, any other 4xx is a request that will fail identically forever. + +Measured 2026-09-16 on discovery-lab: a Tangle Sandbox create refused `HTTP 400 {"code":"CONFIG_ERROR"}`, for a key whose budget was fully reserved by an existing box, retried 14-22 times per node. Eleven of twelve roots showed a durable admission intent and no other event for 25 minutes, with healthy coordination servers and no error anywhere for an operator to read. + +Only a thrown `Error` is read: `status` is an ordinary field name on settlements and run-state records, and treating one as an HTTP refusal stops retries that have nothing to do with a rejected request. + ## 0.237.0 **A refused spawn says which budget channels ran short and by how much.** A `budget-exhausted` reservation now carries `shortfalls`: every channel that did not fit, each as `{ channel, requested, free }` (`ReservationShortfall`, exported). `scope.spawn` passes them through, and `spawn_worker` returns them with a reason a driver can act on, for example `the run pool refused this spawn: iterations has 58 free (this spawn asked for budget.maxIterations 100); budget.maxIterations at most 58 fits`. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 49a19289..081d7a01 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.237.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.237.1` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 541a0e00..016f0f23 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.237.0.** +> **Version 0.237.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.182.0 <0.183.0`. > `sandbox` must satisfy `>=0.36.4 <0.42.0`. diff --git a/package.json b/package.json index 8413137c..a62289ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.237.0", + "version": "0.237.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/runtime/supervise/driver-retry.test.ts b/src/runtime/supervise/driver-retry.test.ts index ddea652a..b665d4b1 100644 --- a/src/runtime/supervise/driver-retry.test.ts +++ b/src/runtime/supervise/driver-retry.test.ts @@ -85,6 +85,68 @@ describe('classifyDriverFailure', () => { expect(classifyDriverFailure(missing)).toBe('terminal') }) + it('reads an HTTP status off a provider SDK error, so a refused create does not retry forever', () => { + // A provider SDK throws its own classes, so a refused environment `create` is neither a + // BackendTransportError nor an AgentEvalError. Measured 2026-09-16: a Tangle Sandbox create + // refused HTTP 400 CONFIG_ERROR (the key's budget was fully reserved by a live box) retried + // 14-22 times per node, and eleven of twelve roots showed a durable intent and nothing else + // for 25 minutes. + class SandboxSdkError extends Error { + constructor( + message: string, + readonly status: number, + readonly code: string, + ) { + super(message) + this.name = 'SandboxError' + } + } + + const refused = new SandboxSdkError('Platform key delegation failed', 400, 'CONFIG_ERROR') + const quota = new SandboxSdkError('concurrent limit reached', 403, 'QUOTA_EXCEEDED') + const wrongState = new SandboxSdkError('sandbox is stopped', 409, 'INVALID_STATE') + expect(classifyDriverFailure(refused)).toBe('terminal') + expect(classifyDriverFailure(quota)).toBe('terminal') + expect(classifyDriverFailure(wrongState)).toBe('terminal') + + // The same split the transport branch promises: the upstream having a bad moment still retries. + expect(classifyDriverFailure(new SandboxSdkError('gateway', 502, 'UPSTREAM'))).toBe('transient') + expect(classifyDriverFailure(new SandboxSdkError('slow down', 429, 'RATE_LIMIT'))).toBe( + 'transient', + ) + expect(classifyDriverFailure(new SandboxSdkError('timeout', 408, 'TIMEOUT'))).toBe('transient') + }) + + it('keeps the historical default when a status is absent or is not an HTTP status', () => { + // A `status` field is common on unrelated objects; only a plain integer in the HTTP range + // decides anything, and everything else keeps retrying as it always did. + expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 'running' }))).toBe( + 'transient', + ) + expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 0 }))).toBe('transient') + expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 302 }))).toBe( + 'transient', + ) + expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 404.5 }))).toBe( + 'transient', + ) + expect(classifyDriverFailure('a thrown string')).toBe('transient') + + // Only a thrown Error is read. `status` is an ordinary field name on settlements, run states + // and provider-model records; a bridge child that aborts mid-turn rejects with such a value, + // and reading it as an HTTP refusal stopped the retry that journals its paid usage. + expect(classifyDriverFailure({ status: 400, kind: 'cancelled' })).toBe('transient') + + // A status behind a throwing getter must not take the classifier down with it. + const hostile = new Error('hostile') + Object.defineProperty(hostile, 'status', { + get() { + throw new Error('status getter exploded') + }, + }) + expect(classifyDriverFailure(hostile)).toBe('transient') + }) + it('settles a profile that cannot materialize after one attempt, with or without a status', () => { // The bridge reports a materialization failure as its own `parse_error` class; on the stream // path no status rides with it, and the status split alone re-drove the same deterministic diff --git a/src/runtime/supervise/driver-retry.ts b/src/runtime/supervise/driver-retry.ts index 69ad4ce6..898e8b9f 100644 --- a/src/runtime/supervise/driver-retry.ts +++ b/src/runtime/supervise/driver-retry.ts @@ -54,7 +54,7 @@ import { ValidationError, } from '../../errors' import { sleep } from '../util' -import { errMessage, errorProperty, errorText } from './error-message' +import { errMessage, errorHttpStatus, errorProperty, errorText } from './error-message' import type { Scope } from './types' /** The scope's live conserved-pool readout — the retry's real bound. Indexed off `Scope` so this @@ -282,7 +282,34 @@ export function classifyDriverFailure( // Every other AgentEvalError (session mismatch, planner, analyst, not-found) is a structural // refusal too. Kept after the transport check, which is itself an AgentEvalError subclass. if (error instanceof AgentEvalError) return 'terminal' - return 'transient' + return foreignHttpStatusVerdict(error) ?? 'transient' +} + +/** + * A foreign error that still carries an HTTP status is a refusal we can read. + * + * A provider SDK throws its own error classes, so an environment `create` that the platform + * refused arrives here as neither a `BackendTransportError` nor an `AgentEvalError`, and the + * default sends it to `transient`. Measured 2026-09-16: a Tangle Sandbox create refused + * `HTTP 400 {"code":"CONFIG_ERROR"}` for a key whose budget was fully reserved retried 14-22 + * times per node while the run showed a durable intent and no other event, so eleven of twelve + * roots sat for 25 minutes with nothing to diagnose. The status was on the error the whole time. + * + * Reading it applies the same rule the transport branch already promises consumers: 408, 429 and + * 5xx are the upstream having a bad moment, and any other 4xx is a request that will fail + * identically forever. Anything without a plain numeric status keeps the historical default. + */ +function foreignHttpStatusVerdict(error: unknown): 'transient' | 'terminal' | undefined { + // Only a thrown Error is read. `status` is a common field name on ordinary objects — a + // settlement, a run state, a provider-model record — and treating one of those as an HTTP + // refusal would silently stop retries that have nothing to do with a rejected request. + // `errorHttpStatus` is the same reader the persisted message uses, so a failure is classified + // from exactly the status an operator will see quoted back to them. + if (!(error instanceof Error)) return undefined + const status = errorHttpStatus(error) + if (status === undefined || status < 400) return undefined + if (status === 408 || status === 429 || status >= 500) return 'transient' + return 'terminal' } /** The budget's own verdict on whether another attempt may run at all. */ diff --git a/src/runtime/supervise/error-message.ts b/src/runtime/supervise/error-message.ts index 4b0bf4be..65dcbcc1 100644 --- a/src/runtime/supervise/error-message.ts +++ b/src/runtime/supervise/error-message.ts @@ -29,7 +29,10 @@ export function errorProperty( } } -function errorHttpStatus(error: Error): number | undefined { +/** An HTTP status carried on a thrown error, read without trusting its getter. Provider SDKs put + * one on their own error classes, so this is how a refusal is recognised outside the transport + * taxonomy — for the persisted message here, and for the driver's retry verdict. */ +export function errorHttpStatus(error: Error): number | undefined { try { const status: unknown = Reflect.get(error, 'status') return typeof status === 'number' && Number.isInteger(status) && status >= 100 && status <= 599 diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index fb091f82..95838906 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:0f561fbee888ec93525d042d6d8099f2e5f9b0d83b412c31fca4cbd1c68ec158", + "digest": "sha256:5a54b47ba429c393a6e74a4e1684a14a425c49d4383db5d1b981234228898172", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.237.0" + "runtimeVersion": "0.237.1" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:d40420c51a75f71cc822c8f128686f8805ae24798f640abd5c4af7a4d15e3d2d", - "runId": "agent-runtime-0.237.0-proposal-fixture", + "recordDigest": "sha256:077382213a151da883e2eb13768ca1e29acff182888f62a594d4f4572cbf3f19", + "runId": "agent-runtime-0.237.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.237.0-proposal-fixture" + "runId": "agent-runtime-0.237.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 5c08dc83..23c7cd2e 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:e71e0b237230e260aba3bcb6d0512ccc08c28c46fe09c892567ece0fd33a424d", + "digest": "sha256:7c366ecdfe20c7a826cd48c9ab06b8ceb5b0ba422fd434ca47fb536c8564fcdd", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.237.0" + "runtimeVersion": "0.237.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:2a1ba8b4e4e8720c2f5823031c7bc9d4e67ae5e21c6c1090a2981e0d204b91e0", + "recordDigest": "sha256:a997ac0ed181839a5cd06293e117409c90f25a9733ba754dbc87aa7e475e7d49", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }