fix(supervise): fail closed when a provider refuses an environment create

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.237.1
+
+**A refused environment create fails closed instead of retrying forever.** A provider SDK throws its own error classes, so a `create` the platform refused arrived at the driver's classifier as neither a `BackendTransportError` nor an `AgentEvalError` and took the foreign-accident default: retry. `classifyDriverFailure` now reads a plain HTTP status off any thrown `Error` and applies the split the transport branch already promises — 408, 429 and 5xx are the upstream having a bad moment, any other 4xx is a request that will fail identically forever.
+
+Measured 2026-09-16 on discovery-lab: a Tangle Sandbox create refused `HTTP 400 {"code":"CONFIG_ERROR"}`, for a key whose budget was fully reserved by an existing box, retried 14-22 times per node. Eleven of twelve roots showed a durable admission intent and no other event for 25 minutes, with healthy coordination servers and no error anywhere for an operator to read.
+
+Only a thrown `Error` is read: `status` is an ordinary field name on settlements and run-state records, and treating one as an HTTP refusal stops retries that have nothing to do with a rejected request.
+
 ## 0.237.0
 
 **A refused spawn says which budget channels ran short and by how much.** A `budget-exhausted` reservation now carries `shortfalls`: every channel that did not fit, each as `{ channel, requested, free }` (`ReservationShortfall`, exported). `scope.spawn` passes them through, and `spawn_worker` returns them with a reason a driver can act on, for example `the run pool refused this spawn: iterations has 58 free (this spawn asked for budget.maxIterations 100); budget.maxIterations at most 58 fits`.

docs/api/primitive-catalog.md

@@ -7,7 +7,7 @@
 
 # Primitive catalog — the never-stale anti-reinvention inventory
 
-> **GENERATED** from `@tangle-network/agent-runtime@0.237.0` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`.
+> **GENERATED** from `@tangle-network/agent-runtime@0.237.1` and `@tangle-network/agent-eval@0.182.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`.
 
 ## 1. agent-runtime — own public surface
 

docs/canonical-api.md

@@ -4,7 +4,7 @@
 Generated signatures and the complete export list live in docs/api/.
 Run pnpm docs:freshness after editing this file. -->
 
-> **Version 0.237.0.**
+> **Version 0.237.1.**
 > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path.
 > `agent-eval` must satisfy `>=0.182.0 <0.183.0`.
 > `sandbox` must satisfy `>=0.36.4 <0.42.0`.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-runtime",
-  "version": "0.237.0",
+  "version": "0.237.1",
   "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.",
   "homepage": "https://github.com/tangle-network/agent-runtime#readme",
   "repository": {

src/runtime/supervise/driver-retry.test.ts

@@ -85,6 +85,68 @@ describe('classifyDriverFailure', () => {
     expect(classifyDriverFailure(missing)).toBe('terminal')
   })
 
+  it('reads an HTTP status off a provider SDK error, so a refused create does not retry forever', () => {
+    // A provider SDK throws its own classes, so a refused environment `create` is neither a
+    // BackendTransportError nor an AgentEvalError. Measured 2026-09-16: a Tangle Sandbox create
+    // refused HTTP 400 CONFIG_ERROR (the key's budget was fully reserved by a live box) retried
+    // 14-22 times per node, and eleven of twelve roots showed a durable intent and nothing else
+    // for 25 minutes.
+    class SandboxSdkError extends Error {
+      constructor(
+        message: string,
+        readonly status: number,
+        readonly code: string,
+      ) {
+        super(message)
+        this.name = 'SandboxError'
+      }
+    }
+
+    const refused = new SandboxSdkError('Platform key delegation failed', 400, 'CONFIG_ERROR')
+    const quota = new SandboxSdkError('concurrent limit reached', 403, 'QUOTA_EXCEEDED')
+    const wrongState = new SandboxSdkError('sandbox is stopped', 409, 'INVALID_STATE')
+    expect(classifyDriverFailure(refused)).toBe('terminal')
+    expect(classifyDriverFailure(quota)).toBe('terminal')
+    expect(classifyDriverFailure(wrongState)).toBe('terminal')
+
+    // The same split the transport branch promises: the upstream having a bad moment still retries.
+    expect(classifyDriverFailure(new SandboxSdkError('gateway', 502, 'UPSTREAM'))).toBe('transient')
+    expect(classifyDriverFailure(new SandboxSdkError('slow down', 429, 'RATE_LIMIT'))).toBe(
+      'transient',
+    )
+    expect(classifyDriverFailure(new SandboxSdkError('timeout', 408, 'TIMEOUT'))).toBe('transient')
+  })
+
+  it('keeps the historical default when a status is absent or is not an HTTP status', () => {
+    // A `status` field is common on unrelated objects; only a plain integer in the HTTP range
+    // decides anything, and everything else keeps retrying as it always did.
+    expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 'running' }))).toBe(
+      'transient',
+    )
+    expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 0 }))).toBe('transient')
+    expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 302 }))).toBe(
+      'transient',
+    )
+    expect(classifyDriverFailure(Object.assign(new Error('died'), { status: 404.5 }))).toBe(
+      'transient',
+    )
+    expect(classifyDriverFailure('a thrown string')).toBe('transient')
+
+    // Only a thrown Error is read. `status` is an ordinary field name on settlements, run states
+    // and provider-model records; a bridge child that aborts mid-turn rejects with such a value,
+    // and reading it as an HTTP refusal stopped the retry that journals its paid usage.
+    expect(classifyDriverFailure({ status: 400, kind: 'cancelled' })).toBe('transient')
+
+    // A status behind a throwing getter must not take the classifier down with it.
+    const hostile = new Error('hostile')
+    Object.defineProperty(hostile, 'status', {
+      get() {
+        throw new Error('status getter exploded')
+      },
+    })
+    expect(classifyDriverFailure(hostile)).toBe('transient')
+  })
+
   it('settles a profile that cannot materialize after one attempt, with or without a status', () => {
     // The bridge reports a materialization failure as its own `parse_error` class; on the stream
     // path no status rides with it, and the status split alone re-drove the same deterministic

src/runtime/supervise/driver-retry.ts

@@ -54,7 +54,7 @@ import {
   ValidationError,
 } from '../../errors'
 import { sleep } from '../util'
-import { errMessage, errorProperty, errorText } from './error-message'
+import { errMessage, errorHttpStatus, errorProperty, errorText } from './error-message'
 import type { Scope } from './types'
 
 /** The scope's live conserved-pool readout — the retry's real bound. Indexed off `Scope` so this
@@ -282,7 +282,34 @@ export function classifyDriverFailure(
   // Every other AgentEvalError (session mismatch, planner, analyst, not-found) is a structural
   // refusal too. Kept after the transport check, which is itself an AgentEvalError subclass.
   if (error instanceof AgentEvalError) return 'terminal'
-  return 'transient'
+  return foreignHttpStatusVerdict(error) ?? 'transient'
+}
+
+/**
+ * A foreign error that still carries an HTTP status is a refusal we can read.
+ *
+ * A provider SDK throws its own error classes, so an environment `create` that the platform
+ * refused arrives here as neither a `BackendTransportError` nor an `AgentEvalError`, and the
+ * default sends it to `transient`. Measured 2026-09-16: a Tangle Sandbox create refused
+ * `HTTP 400 {"code":"CONFIG_ERROR"}` for a key whose budget was fully reserved retried 14-22
+ * times per node while the run showed a durable intent and no other event, so eleven of twelve
+ * roots sat for 25 minutes with nothing to diagnose. The status was on the error the whole time.
+ *
+ * Reading it applies the same rule the transport branch already promises consumers: 408, 429 and
+ * 5xx are the upstream having a bad moment, and any other 4xx is a request that will fail
+ * identically forever. Anything without a plain numeric status keeps the historical default.
+ */
+function foreignHttpStatusVerdict(error: unknown): 'transient' | 'terminal' | undefined {
+  // Only a thrown Error is read. `status` is a common field name on ordinary objects — a
+  // settlement, a run state, a provider-model record — and treating one of those as an HTTP
+  // refusal would silently stop retries that have nothing to do with a rejected request.
+  // `errorHttpStatus` is the same reader the persisted message uses, so a failure is classified
+  // from exactly the status an operator will see quoted back to them.
+  if (!(error instanceof Error)) return undefined
+  const status = errorHttpStatus(error)
+  if (status === undefined || status < 400) return undefined
+  if (status === 408 || status === 429 || status >= 500) return 'transient'
+  return 'terminal'
 }
 
 /** The budget's own verdict on whether another attempt may run at all. */

src/runtime/supervise/error-message.ts

@@ -29,7 +29,10 @@ export function errorProperty(
   }
 }
 
-function errorHttpStatus(error: Error): number | undefined {
+/** An HTTP status carried on a thrown error, read without trusting its getter. Provider SDKs put
+ *  one on their own error classes, so this is how a refusal is recognised outside the transport
+ *  taxonomy — for the persisted message here, and for the driver's retry verdict. */
+export function errorHttpStatus(error: Error): number | undefined {
   try {
     const status: unknown = Reflect.get(error, 'status')
     return typeof status === 'number' && Number.isInteger(status) && status >= 100 && status <= 599

src/testing/fixtures/agent-improvement-proposal.json

@@ -1,6 +1,6 @@
 {
   "changedSurfaces": ["prompt"],
-  "digest": "sha256:0f561fbee888ec93525d042d6d8099f2e5f9b0d83b412c31fca4cbd1c68ec158",
+  "digest": "sha256:5a54b47ba429c393a6e74a4e1684a14a425c49d4383db5d1b981234228898172",
   "evaluation": {
     "decision": {
       "contributingChecks": [
@@ -4882,7 +4882,7 @@
     ],
     "metadata": {
       "fixture": "agent-improvement-proposal",
-      "runtimeVersion": "0.237.0"
+      "runtimeVersion": "0.237.1"
     },
     "objectives": [
       {
@@ -4993,8 +4993,8 @@
       "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09",
       "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693",
       "kind": "agent-eval-loop",
-      "recordDigest": "sha256:d40420c51a75f71cc822c8f128686f8805ae24798f640abd5c4af7a4d15e3d2d",
-      "runId": "agent-runtime-0.237.0-proposal-fixture",
+      "recordDigest": "sha256:077382213a151da883e2eb13768ca1e29acff182888f62a594d4f4572cbf3f19",
+      "runId": "agent-runtime-0.237.1-proposal-fixture",
       "schema": "agent-candidate-experiment"
     }
   },
@@ -5021,5 +5021,5 @@
   ],
   "kind": "agent-improvement-proposal",
   "proposedAt": "2026-07-10T01:00:00.000Z",
-  "runId": "agent-runtime-0.237.0-proposal-fixture"
+  "runId": "agent-runtime-0.237.1-proposal-fixture"
 }

src/testing/fixtures/agent-profile-improvement-proposal.json

@@ -1,6 +1,6 @@
 {
   "changedSurfaces": ["prompt", "skills"],
-  "digest": "sha256:e71e0b237230e260aba3bcb6d0512ccc08c28c46fe09c892567ece0fd33a424d",
+  "digest": "sha256:7c366ecdfe20c7a826cd48c9ab06b8ceb5b0ba422fd434ca47fb536c8564fcdd",
   "evaluation": {
     "decision": {
       "contributingChecks": [
@@ -1715,7 +1715,7 @@
     ],
     "metadata": {
       "fixture": "agent-profile-improvement-proposal",
-      "runtimeVersion": "0.237.0"
+      "runtimeVersion": "0.237.1"
     },
     "objectives": [
       {
@@ -1826,7 +1826,7 @@
       "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704",
       "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9",
       "kind": "agent-eval-loop",
-      "recordDigest": "sha256:2a1ba8b4e4e8720c2f5823031c7bc9d4e67ae5e21c6c1090a2981e0d204b91e0",
+      "recordDigest": "sha256:a997ac0ed181839a5cd06293e117409c90f25a9733ba754dbc87aa7e475e7d49",
       "runId": "profile-improvement-1",
       "schema": "agent-profile-improvement-experiment"
     }