queen: a refused key rests and the next bee takes the next key (key_failover card)

trios/agent-server/apps/server/src/api/services/queen-bee-actors-round.ts

@@ -78,7 +78,10 @@ export interface RoundHandoff {
  */
 export function beeEndOfRow(outcome: string | null): BeeEnd {
   const o = outcome ?? ''
-  const provider = o.startsWith(DISPATCH_OUTCOME_LABELS.providerQuotaExhausted)
+  const provider = [
+    DISPATCH_OUTCOME_LABELS.providerQuotaExhausted,
+    DISPATCH_OUTCOME_LABELS.providerOverloaded,
+  ].some((p) => o.startsWith(p))
   const reaped = outcome === null || o.startsWith('reaped')
   return {
     completion: o === DISPATCH_OUTCOME_LABELS.finished,

trios/agent-server/apps/server/src/api/services/queen-contributor-keys.ts

@@ -7,7 +7,10 @@ import {
 } from 'node:crypto'
 import type { Pool } from 'pg'
 import { logger } from '../../lib/logger'
+import { loadCardWasm, u32 } from './queen-card-wasm'
 import { CONTRIBUTOR_POLICY as P } from './queen-contributor-policy'
+import * as KF from './queen-key-failover-card.gen'
+import { RUNNER_KEY_BASE } from './queen-runners'
 
 export type ContributorProvider = 'nvidia' | 'zai'
 export type ProbeStatus =
@@ -1034,3 +1037,27 @@ export async function contributorOwnerNames(
   )
   return Object.fromEntries(rows.map((row) => [row.key_index, row.owner_name]))
 }
+
+// #9998: keys a refusal rests (key_failover card), read from the closed rows.
+export const failover = () => loadCardWasm('queen/key_failover.wasm')
+export const keyFailoverOn = () =>
+  process.env.TRIOS_QUEEN_KEY_FAILOVER === 'on'
+export async function restingKeys(db: Pool): Promise<number[]> {
+  if (!keyFailoverOn()) return []
+  const { rows } = await db.query(
+    `SELECT key_index k, bool_or(outcome LIKE 'provider quota%') q,
+            (count(*) - 1)::int s,
+            min(EXTRACT(EPOCH FROM now() - finished_at))::int a
+       FROM queen_dispatch
+      WHERE outcome ~ '^provider (overloaded|quota)' AND key_index <= $2
+        AND finished_at > now() - $1 * interval '1 second'
+      GROUP BY key_index`,
+    [KF.KEY_BREAK_SECONDS, RUNNER_KEY_BASE],
+  )
+  return rows
+    .filter((r) => {
+      const kind = r.q ? KF.KE_QUOTA : KF.KE_TRANSIENT
+      return r.a < failover().call('cooldown_seconds', kind, u32(r.s))
+    })
+    .map((r) => r.k)
+}

trios/agent-server/apps/server/src/api/services/queen-dispatch.ts

@@ -29,10 +29,14 @@ import { statSync } from 'node:fs'
 import type { Pool } from 'pg'
 import { logger } from '../../lib/logger'
 import { shellArgv } from '../../tools/filesystem/bash'
+import { u32 } from './queen-card-wasm'
 import {
   type ContributorRuntime,
   contributorRuntime,
   type EnvironmentKey,
+  failover,
+  keyFailoverOn,
+  restingKeys,
 } from './queen-contributor-keys'
 import {
   claimTaskLease,
@@ -48,6 +52,7 @@ import {
   TIP_SENT_BACK,
 } from './queen-control-rules'
 import { domainOfTask } from './queen-domains'
+import * as KF from './queen-key-failover-card.gen'
 import { queenActorHolderPrefix, queenHolderName } from './queen-lease'
 import {
   describeReading,
@@ -132,6 +137,8 @@ export const DISPATCH_OUTCOME_LABELS = {
    * 1308)`. The response body itself stays off the row.
    */
   providerQuotaExhausted: 'provider quota exhausted',
+  /** Base of the key_failover ending (#9998): `(zai code N)` or `(http N)`. */
+  providerOverloaded: 'provider overloaded',
   /**
    * Base of the boot reaper's ending; the writer appends the explanation.
    * Must keep the `reaped` prefix - queen-tick and queen-kanban match on it.
@@ -4171,40 +4178,9 @@ class Scribe {
   }
 }
 
-/**
- * #1301. Z.ai business codes whose documented meaning is that a quota
- * window, a plan, or a balance is spent (docs.z.ai, "Errors": every one of
- * them arrives as HTTP 429).
- *
- * Coding Plan removed the synthetic USD start gate (#1300), which makes these
- * responses the authoritative stop signal: a bee that hits one cannot be
- * helped by another retry until the provider resets the window or the
- * operator pays. A generic worker failure hides that distinction, so the
- * dispatch boundary keeps it as a closed list.
- *
- * The transient 429s are deliberately NOT here. 1302 is a request-rate limit
- * and 1305 a temporary overload; both clear on their own, and closing a bee
- * as quota-stopped over either would retire work another retry could have
- * finished. `1113` is also matched at the transport
- * (lib/provider-error-classifier.ts) to stop SDK retries; this list is about
- * the ending, not the retry.
- */
-const ZAI_QUOTA_EXHAUSTED_CODES: ReadonlySet<string> = new Set([
-  '1113', // Insufficient balance or no resource package. Please recharge.
-  '1308', // Usage limit reached for a window; resets at a stated time.
-  '1309', // GLM Coding Plan package expired.
-  '1310', // Weekly/monthly limit exhausted.
-  '1311', // Subscription plan does not include the model.
-  '1313', // Fair Usage Policy limit on the account.
-  '1314', // Enterprise package expired.
-  '1315', // Key limited to enterprise coding package scenarios.
-  '1316', // 5-hour usage limit; no balance for extra usage.
-  '1317', // 7-day usage limit; no balance for extra usage.
-  '1318', // 5-hour usage limit; monthly spend limit reached.
-  '1319', // 7-day usage limit; monthly spend limit reached.
-  '1320', // 5-hour usage limit; monthly spend limit reached.
-  '1321', // 7-day usage limit; monthly spend limit reached.
-])
+// #1301. The closed list of Z.ai quota codes (1113, 1308-1321 except 1312) is
+// `is_quota_code` in specs/queen/key_failover.t27 (#9998); 1302 and 1305 are
+// transient and not on it.
 
 /**
  * The one ending a quota-stopped bee closes with (#1301).
@@ -4224,7 +4200,7 @@ export function classifyQuotaExhaustion(
   // the Coding Plan window this classification names belongs to Z.ai alone.
   if (provider !== 'zai') return null
   for (const code of zaiCodesIn(errorText)) {
-    if (ZAI_QUOTA_EXHAUSTED_CODES.has(code)) {
+    if (failover().call('is_quota_code', u32(Number(code))) !== 0) {
       // The label base is enumerated with every other outcome (#1360); the
       // code is the one closed parameter it may carry.
       return `${DISPATCH_OUTCOME_LABELS.providerQuotaExhausted} (zai code ${code})`
@@ -4233,6 +4209,18 @@ export function classifyQuotaExhaustion(
   return null
 }
 
+/** #9998: a refused key (429, 5xx, Z.ai 1302/1305) closes as an overload. */
+export function classifyOverload(provider: string, text: string) {
+  const status = Number(/HTTP (\d{3})/.exec(text)?.[1] ?? 0)
+  const code = Number((provider === 'zai' && zaiCodesIn(text)[0]) || 0)
+  const zai = code === KF.ZAI_OVERLOAD || code === KF.ZAI_RATE_LIMIT
+  if (!keyFailoverOn() || !(status || zai)) return null
+  const kind = failover().call('error_kind', u32(status), u32(code))
+  if (kind !== KF.KE_TRANSIENT) return null
+  const why = zai ? `zai code ${code}` : `http ${status}`
+  return `${DISPATCH_OUTCOME_LABELS.providerOverloaded} (${why})`
+}
+
 /**
  * The Z.ai business codes a turn's terminal error text carries.
  *
@@ -4339,7 +4327,8 @@ export async function drain(
     scribe.streamError !== undefined
   ) {
     const quota = classifyQuotaExhaustion(provider, scribe.streamError)
-    if (quota !== null) outcome = quota
+    const rested = classifyOverload(provider, scribe.streamError)
+    outcome = quota ?? rested ?? outcome
   }
   // #1360. A stream that closed without ever signalling completion ended in a
   // way nobody measured: something - a tool event, a truncation, anything -
@@ -4975,7 +4964,15 @@ export async function dispatchBee(
   }
 
   const runtime = await contributorRuntime(pool, environmentContributorKeys())
-  const chosen = resolveWorkerProvider(takenKeyIndices, afterKeyIndex, runtime)
+  // #9998: a resting key counts as full, so the bee takes the next key.
+  const resting = (await restingKeys(pool)).flatMap((key) =>
+    Array<number>(KF.POOL_MAX).fill(key),
+  )
+  const chosen = resolveWorkerProvider(
+    [...takenKeyIndices, ...resting],
+    afterKeyIndex,
+    runtime,
+  )
   if (chosen?.exhausted !== undefined) {
     // Not a missing credential: every key this deployment has is already
     // carrying a bee. Named separately because the fix is different - one more

trios/agent-server/apps/server/src/api/services/queen-key-failover-card.gen.ts

@@ -0,0 +1,64 @@
+// Generated by `t27c gen-ts` from key_failover.t27. Do not edit.
+//
+// Every name, value and type here comes from that spec. Change the spec and
+// regenerate; an edit made here is lost the next time anyone builds.
+
+export const KE_OK = 0 satisfies number;
+export const KE_TRANSIENT = 1 satisfies number;
+export const KE_QUOTA = 2 satisfies number;
+export const KE_AUTH = 3 satisfies number;
+export const KE_REQUEST = 4 satisfies number;
+export const ZAI_RATE_LIMIT = 1302 satisfies number;
+export const ZAI_OVERLOAD = 1305 satisfies number;
+// t27c gen-ts: fn is_quota_code was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: fn error_kind was not emitted -- this backend lowers declarations, not bodies.
+export const COOL_BASE_SECONDS = 15 satisfies number;
+export const COOL_CAP_SECONDS = 240 satisfies number;
+export const KEY_BREAK_SECONDS = 1800 satisfies number;
+// t27c gen-ts: fn cooldown_seconds was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: fn key_closes was not emitted -- this backend lowers declarations, not bodies.
+export const POOL_MAX = 32 satisfies number;
+export const NO_KEY = 255 satisfies number;
+// t27c gen-ts: fn bit was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: fn is_set was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: fn next_free was not emitted -- this backend lowers declarations, not bodies.
+export const POOL_OVERLOAD_KEYS = 3 satisfies number;
+export const OVERLOAD_WINDOW_SECONDS = 60 satisfies number;
+// t27c gen-ts: fn pool_overloaded was not emitted -- this backend lowers declarations, not bodies.
+export const FA_DONE = 0 satisfies number;
+export const FA_NEXT_KEY = 1 satisfies number;
+export const FA_NEXT_POOL = 2 satisfies number;
+export const FA_GIVE_UP = 3 satisfies number;
+export const FA_FAIL = 4 satisfies number;
+export const MAX_ATTEMPTS_PER_TURN = 8 satisfies number;
+// t27c gen-ts: fn step was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: fn charges_issue was not emitted -- this backend lowers declarations, not bodies.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact.
+// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact.
+
+// Declaration order, which the spec's own laws depend on.
+export const __STRUCT_ORDER__ = [] as const;
+export const __DECL_ORDER__ = ["KE_OK", "KE_TRANSIENT", "KE_QUOTA", "KE_AUTH", "KE_REQUEST", "ZAI_RATE_LIMIT", "ZAI_OVERLOAD", "COOL_BASE_SECONDS", "COOL_CAP_SECONDS", "KEY_BREAK_SECONDS", "POOL_MAX", "NO_KEY", "POOL_OVERLOAD_KEYS", "OVERLOAD_WINDOW_SECONDS", "FA_DONE", "FA_NEXT_KEY", "FA_NEXT_POOL", "FA_GIVE_UP", "FA_FAIL", "MAX_ATTEMPTS_PER_TURN"] as const;
+
+// What this spec holds and this backend did not print. Empty is the whole
+// story most of the time; an entry here is a promise the artifact does not
+// keep, and reading it is how a tool tells a partial module from a complete
+// one without parsing comments.
+//
+// A LIST, not a map keyed by name. `specs/numeric/formats.t27` declares four
+// separate consts called `result`, one per test block, and under an object
+// literal three of the four omissions vanished into the fourth -- a record of
+// what went missing that itself went missing. A spec is free to reuse a
+// name; this file is not free to lose the second one.
+export const __NOT_EMITTED__ = Object.freeze([] as const);

trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts

@@ -2158,42 +2158,15 @@ describe('classifying a quota stop at the dispatch boundary', () => {
   })
 
   it('classifies every documented quota code, and only those', () => {
-    const quotaCodes = [
-      '1113', // Insufficient balance or no resource package.
-      '1308', // Usage limit reached for a window.
-      '1309', // GLM Coding Plan package expired.
-      '1310', // Weekly/Monthly limit exhausted.
-      '1311', // Plan does not include the model.
-      '1313', // Fair Usage Policy.
-      '1314', // Enterprise package expired.
-      '1315', // Key limited to enterprise coding package.
-      '1316', // 5-hour limit, no balance for extra usage.
-      '1317', // 7-day limit, no balance for extra usage.
-      '1318', // 5-hour limit, monthly spend limit.
-      '1319', // 7-day limit, monthly spend limit.
-      '1320', // 5-hour limit, monthly spend limit.
-      '1321', // 7-day limit, monthly spend limit.
+    const quota = [
+      1113, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319,
+      1320, 1321,
     ]
-    for (const code of quotaCodes) {
+    for (const code of [...quota, 1000, 1211, 1220, 1301, 1302, 1305]) {
       expect(classifyQuotaExhaustion('zai', `[${code}] stopped`)).toBe(
-        `provider quota exhausted (zai code ${code})`,
+        quota.includes(code) ? `provider quota exhausted (zai code ${code})` : null,
       )
     }
-    // The transient 429s clear on their own. Closing a bee as quota-stopped
-    // over either would retire work another retry could have finished.
-    expect(
-      classifyQuotaExhaustion('zai', '[1302] Rate limit reached'),
-    ).toBeNull()
-    expect(
-      classifyQuotaExhaustion(
-        'zai',
-        '[1305] The service may be temporarily overloaded',
-      ),
-    ).toBeNull()
-    // Documented codes outside the quota family are not quota stops either.
-    for (const code of ['1000', '1211', '1220', '1301']) {
-      expect(classifyQuotaExhaustion('zai', `[${code}] other`)).toBeNull()
-    }
   })
 
   // The message this server's own transport builds is `[code] message`

trios/agent-server/specs/PIN

@@ -127,3 +127,6 @@ claim card: gHashTag/t27 specs/queen/claim.t27 (t27#8765, PR t27#8768 head d4ced
 claim: queen/claim.t27 sha256 f031594c8c0c5078a126ad4ed3ec0e3ef6b1179658572878fa24361c4b17cec9
 claim: queen/review_valve.t27 sha256 5177806d264aa3611b5187a89b929c2231cbf7f95eb3cb78d246d913b658d8a6
 claim: queen/claim.wasm sha256 08fc2b32d46742638c00d20fe1e64dd81289080dcfc8fa662972acc411d4884a
+key_failover card: gHashTag/t27 specs/queen/key_failover.t27 (t27#9998, Closes t27#9993; wired by t27#10403), byte-identical; key_failover.wasm = t27c gen-c + zig 0.16.0 cc --target=wasm32-wasi -Oz -DNDEBUG -mexec-model=reactor, exports every fn of the card: is_quota_code,error_kind,cooldown_seconds,key_closes,bit,is_set,next_free,pool_overloaded,step,charges_issue (reproducible: two builds, one hash); constants: t27c gen-ts > apps/server/src/api/services/queen-key-failover-card.gen.ts
+key_failover: queen/key_failover.t27 sha256 822367357fc6b487c1c7fb7ac8369165e6a7e6636fd9b0dcafec29e9a6274301
+key_failover: queen/key_failover.wasm sha256 aa4fdd776ccf74ecff59cee1c5284db16ead03e477afd9988409920afa0b6b4f