diff --git a/trios/agent-server/apps/server/src/api/services/queen-bee-actors-round.ts b/trios/agent-server/apps/server/src/api/services/queen-bee-actors-round.ts index f3ace05af9..55186dada0 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-bee-actors-round.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-bee-actors-round.ts @@ -78,7 +78,10 @@ export interface RoundHandoff { */ export function beeEndOfRow(outcome: string | null): BeeEnd { const o = outcome ?? '' - const provider = o.startsWith(DISPATCH_OUTCOME_LABELS.providerQuotaExhausted) + const provider = [ + DISPATCH_OUTCOME_LABELS.providerQuotaExhausted, + DISPATCH_OUTCOME_LABELS.providerOverloaded, + ].some((p) => o.startsWith(p)) const reaped = outcome === null || o.startsWith('reaped') return { completion: o === DISPATCH_OUTCOME_LABELS.finished, diff --git a/trios/agent-server/apps/server/src/api/services/queen-contributor-keys.ts b/trios/agent-server/apps/server/src/api/services/queen-contributor-keys.ts index d809f32257..79eede7088 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-contributor-keys.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-contributor-keys.ts @@ -7,7 +7,10 @@ import { } from 'node:crypto' import type { Pool } from 'pg' import { logger } from '../../lib/logger' +import { loadCardWasm, u32 } from './queen-card-wasm' import { CONTRIBUTOR_POLICY as P } from './queen-contributor-policy' +import * as KF from './queen-key-failover-card.gen' +import { RUNNER_KEY_BASE } from './queen-runners' export type ContributorProvider = 'nvidia' | 'zai' export type ProbeStatus = @@ -1034,3 +1037,27 @@ export async function contributorOwnerNames( ) return Object.fromEntries(rows.map((row) => [row.key_index, row.owner_name])) } + +// #9998: keys a refusal rests (key_failover card), read from the closed rows. +export const failover = () => loadCardWasm('queen/key_failover.wasm') +export const keyFailoverOn = () => + process.env.TRIOS_QUEEN_KEY_FAILOVER === 'on' +export async function restingKeys(db: Pool): Promise { + if (!keyFailoverOn()) return [] + const { rows } = await db.query( + `SELECT key_index k, bool_or(outcome LIKE 'provider quota%') q, + (count(*) - 1)::int s, + min(EXTRACT(EPOCH FROM now() - finished_at))::int a + FROM queen_dispatch + WHERE outcome ~ '^provider (overloaded|quota)' AND key_index <= $2 + AND finished_at > now() - $1 * interval '1 second' + GROUP BY key_index`, + [KF.KEY_BREAK_SECONDS, RUNNER_KEY_BASE], + ) + return rows + .filter((r) => { + const kind = r.q ? KF.KE_QUOTA : KF.KE_TRANSIENT + return r.a < failover().call('cooldown_seconds', kind, u32(r.s)) + }) + .map((r) => r.k) +} diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 9d759625df..20ee9b3266 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -29,10 +29,14 @@ import { statSync } from 'node:fs' import type { Pool } from 'pg' import { logger } from '../../lib/logger' import { shellArgv } from '../../tools/filesystem/bash' +import { u32 } from './queen-card-wasm' import { type ContributorRuntime, contributorRuntime, type EnvironmentKey, + failover, + keyFailoverOn, + restingKeys, } from './queen-contributor-keys' import { claimTaskLease, @@ -48,6 +52,7 @@ import { TIP_SENT_BACK, } from './queen-control-rules' import { domainOfTask } from './queen-domains' +import * as KF from './queen-key-failover-card.gen' import { queenActorHolderPrefix, queenHolderName } from './queen-lease' import { describeReading, @@ -132,6 +137,8 @@ export const DISPATCH_OUTCOME_LABELS = { * 1308)`. The response body itself stays off the row. */ providerQuotaExhausted: 'provider quota exhausted', + /** Base of the key_failover ending (#9998): `(zai code N)` or `(http N)`. */ + providerOverloaded: 'provider overloaded', /** * Base of the boot reaper's ending; the writer appends the explanation. * Must keep the `reaped` prefix - queen-tick and queen-kanban match on it. @@ -4171,40 +4178,9 @@ class Scribe { } } -/** - * #1301. Z.ai business codes whose documented meaning is that a quota - * window, a plan, or a balance is spent (docs.z.ai, "Errors": every one of - * them arrives as HTTP 429). - * - * Coding Plan removed the synthetic USD start gate (#1300), which makes these - * responses the authoritative stop signal: a bee that hits one cannot be - * helped by another retry until the provider resets the window or the - * operator pays. A generic worker failure hides that distinction, so the - * dispatch boundary keeps it as a closed list. - * - * The transient 429s are deliberately NOT here. 1302 is a request-rate limit - * and 1305 a temporary overload; both clear on their own, and closing a bee - * as quota-stopped over either would retire work another retry could have - * finished. `1113` is also matched at the transport - * (lib/provider-error-classifier.ts) to stop SDK retries; this list is about - * the ending, not the retry. - */ -const ZAI_QUOTA_EXHAUSTED_CODES: ReadonlySet = new Set([ - '1113', // Insufficient balance or no resource package. Please recharge. - '1308', // Usage limit reached for a window; resets at a stated time. - '1309', // GLM Coding Plan package expired. - '1310', // Weekly/monthly limit exhausted. - '1311', // Subscription plan does not include the model. - '1313', // Fair Usage Policy limit on the account. - '1314', // Enterprise package expired. - '1315', // Key limited to enterprise coding package scenarios. - '1316', // 5-hour usage limit; no balance for extra usage. - '1317', // 7-day usage limit; no balance for extra usage. - '1318', // 5-hour usage limit; monthly spend limit reached. - '1319', // 7-day usage limit; monthly spend limit reached. - '1320', // 5-hour usage limit; monthly spend limit reached. - '1321', // 7-day usage limit; monthly spend limit reached. -]) +// #1301. The closed list of Z.ai quota codes (1113, 1308-1321 except 1312) is +// `is_quota_code` in specs/queen/key_failover.t27 (#9998); 1302 and 1305 are +// transient and not on it. /** * The one ending a quota-stopped bee closes with (#1301). @@ -4224,7 +4200,7 @@ export function classifyQuotaExhaustion( // the Coding Plan window this classification names belongs to Z.ai alone. if (provider !== 'zai') return null for (const code of zaiCodesIn(errorText)) { - if (ZAI_QUOTA_EXHAUSTED_CODES.has(code)) { + if (failover().call('is_quota_code', u32(Number(code))) !== 0) { // The label base is enumerated with every other outcome (#1360); the // code is the one closed parameter it may carry. return `${DISPATCH_OUTCOME_LABELS.providerQuotaExhausted} (zai code ${code})` @@ -4233,6 +4209,18 @@ export function classifyQuotaExhaustion( return null } +/** #9998: a refused key (429, 5xx, Z.ai 1302/1305) closes as an overload. */ +export function classifyOverload(provider: string, text: string) { + const status = Number(/HTTP (\d{3})/.exec(text)?.[1] ?? 0) + const code = Number((provider === 'zai' && zaiCodesIn(text)[0]) || 0) + const zai = code === KF.ZAI_OVERLOAD || code === KF.ZAI_RATE_LIMIT + if (!keyFailoverOn() || !(status || zai)) return null + const kind = failover().call('error_kind', u32(status), u32(code)) + if (kind !== KF.KE_TRANSIENT) return null + const why = zai ? `zai code ${code}` : `http ${status}` + return `${DISPATCH_OUTCOME_LABELS.providerOverloaded} (${why})` +} + /** * The Z.ai business codes a turn's terminal error text carries. * @@ -4339,7 +4327,8 @@ export async function drain( scribe.streamError !== undefined ) { const quota = classifyQuotaExhaustion(provider, scribe.streamError) - if (quota !== null) outcome = quota + const rested = classifyOverload(provider, scribe.streamError) + outcome = quota ?? rested ?? outcome } // #1360. A stream that closed without ever signalling completion ended in a // way nobody measured: something - a tool event, a truncation, anything - @@ -4975,7 +4964,15 @@ export async function dispatchBee( } const runtime = await contributorRuntime(pool, environmentContributorKeys()) - const chosen = resolveWorkerProvider(takenKeyIndices, afterKeyIndex, runtime) + // #9998: a resting key counts as full, so the bee takes the next key. + const resting = (await restingKeys(pool)).flatMap((key) => + Array(KF.POOL_MAX).fill(key), + ) + const chosen = resolveWorkerProvider( + [...takenKeyIndices, ...resting], + afterKeyIndex, + runtime, + ) if (chosen?.exhausted !== undefined) { // Not a missing credential: every key this deployment has is already // carrying a bee. Named separately because the fix is different - one more diff --git a/trios/agent-server/apps/server/src/api/services/queen-key-failover-card.gen.ts b/trios/agent-server/apps/server/src/api/services/queen-key-failover-card.gen.ts new file mode 100644 index 0000000000..1fc8bf65c4 --- /dev/null +++ b/trios/agent-server/apps/server/src/api/services/queen-key-failover-card.gen.ts @@ -0,0 +1,64 @@ +// Generated by `t27c gen-ts` from key_failover.t27. Do not edit. +// +// Every name, value and type here comes from that spec. Change the spec and +// regenerate; an edit made here is lost the next time anyone builds. + +export const KE_OK = 0 satisfies number; +export const KE_TRANSIENT = 1 satisfies number; +export const KE_QUOTA = 2 satisfies number; +export const KE_AUTH = 3 satisfies number; +export const KE_REQUEST = 4 satisfies number; +export const ZAI_RATE_LIMIT = 1302 satisfies number; +export const ZAI_OVERLOAD = 1305 satisfies number; +// t27c gen-ts: fn is_quota_code was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn error_kind was not emitted -- this backend lowers declarations, not bodies. +export const COOL_BASE_SECONDS = 15 satisfies number; +export const COOL_CAP_SECONDS = 240 satisfies number; +export const KEY_BREAK_SECONDS = 1800 satisfies number; +// t27c gen-ts: fn cooldown_seconds was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn key_closes was not emitted -- this backend lowers declarations, not bodies. +export const POOL_MAX = 32 satisfies number; +export const NO_KEY = 255 satisfies number; +// t27c gen-ts: fn bit was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn is_set was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn next_free was not emitted -- this backend lowers declarations, not bodies. +export const POOL_OVERLOAD_KEYS = 3 satisfies number; +export const OVERLOAD_WINDOW_SECONDS = 60 satisfies number; +// t27c gen-ts: fn pool_overloaded was not emitted -- this backend lowers declarations, not bodies. +export const FA_DONE = 0 satisfies number; +export const FA_NEXT_KEY = 1 satisfies number; +export const FA_NEXT_POOL = 2 satisfies number; +export const FA_GIVE_UP = 3 satisfies number; +export const FA_FAIL = 4 satisfies number; +export const MAX_ATTEMPTS_PER_TURN = 8 satisfies number; +// t27c gen-ts: fn step was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn charges_issue was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. + +// Declaration order, which the spec's own laws depend on. +export const __STRUCT_ORDER__ = [] as const; +export const __DECL_ORDER__ = ["KE_OK", "KE_TRANSIENT", "KE_QUOTA", "KE_AUTH", "KE_REQUEST", "ZAI_RATE_LIMIT", "ZAI_OVERLOAD", "COOL_BASE_SECONDS", "COOL_CAP_SECONDS", "KEY_BREAK_SECONDS", "POOL_MAX", "NO_KEY", "POOL_OVERLOAD_KEYS", "OVERLOAD_WINDOW_SECONDS", "FA_DONE", "FA_NEXT_KEY", "FA_NEXT_POOL", "FA_GIVE_UP", "FA_FAIL", "MAX_ATTEMPTS_PER_TURN"] as const; + +// What this spec holds and this backend did not print. Empty is the whole +// story most of the time; an entry here is a promise the artifact does not +// keep, and reading it is how a tool tells a partial module from a complete +// one without parsing comments. +// +// A LIST, not a map keyed by name. `specs/numeric/formats.t27` declares four +// separate consts called `result`, one per test block, and under an object +// literal three of the four omissions vanished into the fourth -- a record of +// what went missing that itself went missing. A spec is free to reuse a +// name; this file is not free to lose the second one. +export const __NOT_EMITTED__ = Object.freeze([] as const); diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index bcd2b74ae9..f9c8c46f38 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -2158,42 +2158,15 @@ describe('classifying a quota stop at the dispatch boundary', () => { }) it('classifies every documented quota code, and only those', () => { - const quotaCodes = [ - '1113', // Insufficient balance or no resource package. - '1308', // Usage limit reached for a window. - '1309', // GLM Coding Plan package expired. - '1310', // Weekly/Monthly limit exhausted. - '1311', // Plan does not include the model. - '1313', // Fair Usage Policy. - '1314', // Enterprise package expired. - '1315', // Key limited to enterprise coding package. - '1316', // 5-hour limit, no balance for extra usage. - '1317', // 7-day limit, no balance for extra usage. - '1318', // 5-hour limit, monthly spend limit. - '1319', // 7-day limit, monthly spend limit. - '1320', // 5-hour limit, monthly spend limit. - '1321', // 7-day limit, monthly spend limit. + const quota = [ + 1113, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, + 1320, 1321, ] - for (const code of quotaCodes) { + for (const code of [...quota, 1000, 1211, 1220, 1301, 1302, 1305]) { expect(classifyQuotaExhaustion('zai', `[${code}] stopped`)).toBe( - `provider quota exhausted (zai code ${code})`, + quota.includes(code) ? `provider quota exhausted (zai code ${code})` : null, ) } - // The transient 429s clear on their own. Closing a bee as quota-stopped - // over either would retire work another retry could have finished. - expect( - classifyQuotaExhaustion('zai', '[1302] Rate limit reached'), - ).toBeNull() - expect( - classifyQuotaExhaustion( - 'zai', - '[1305] The service may be temporarily overloaded', - ), - ).toBeNull() - // Documented codes outside the quota family are not quota stops either. - for (const code of ['1000', '1211', '1220', '1301']) { - expect(classifyQuotaExhaustion('zai', `[${code}] other`)).toBeNull() - } }) // The message this server's own transport builds is `[code] message` diff --git a/trios/agent-server/specs/PIN b/trios/agent-server/specs/PIN index c0542c46fc..d0e455611d 100644 --- a/trios/agent-server/specs/PIN +++ b/trios/agent-server/specs/PIN @@ -127,3 +127,6 @@ claim card: gHashTag/t27 specs/queen/claim.t27 (t27#8765, PR t27#8768 head d4ced claim: queen/claim.t27 sha256 f031594c8c0c5078a126ad4ed3ec0e3ef6b1179658572878fa24361c4b17cec9 claim: queen/review_valve.t27 sha256 5177806d264aa3611b5187a89b929c2231cbf7f95eb3cb78d246d913b658d8a6 claim: queen/claim.wasm sha256 08fc2b32d46742638c00d20fe1e64dd81289080dcfc8fa662972acc411d4884a +key_failover card: gHashTag/t27 specs/queen/key_failover.t27 (t27#9998, Closes t27#9993; wired by t27#10403), byte-identical; key_failover.wasm = t27c gen-c + zig 0.16.0 cc --target=wasm32-wasi -Oz -DNDEBUG -mexec-model=reactor, exports every fn of the card: is_quota_code,error_kind,cooldown_seconds,key_closes,bit,is_set,next_free,pool_overloaded,step,charges_issue (reproducible: two builds, one hash); constants: t27c gen-ts > apps/server/src/api/services/queen-key-failover-card.gen.ts +key_failover: queen/key_failover.t27 sha256 822367357fc6b487c1c7fb7ac8369165e6a7e6636fd9b0dcafec29e9a6274301 +key_failover: queen/key_failover.wasm sha256 aa4fdd776ccf74ecff59cee1c5284db16ead03e477afd9988409920afa0b6b4f diff --git a/trios/agent-server/specs/queen/key_failover.t27 b/trios/agent-server/specs/queen/key_failover.t27 new file mode 100644 index 0000000000..3ce96bef2b --- /dev/null +++ b/trios/agent-server/specs/queen/key_failover.t27 @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/queen/key_failover.t27 -- a bee whose key is refused moves to another key, then another pool (gHashTag/t27#9993). +; Measured 2026-10-17 13:00-15:18Z in the Railway logs of trios-bee-runner-2: 93 bee turns, 17 of them +; ended after 0 steps; at least 22 AI_RetryError on z.ai "429 code 1305" (temporary overload), in bursts +; at 13:53, 13:59, 14:08 (16 in one minute), 14:19, 14:47 and 14:50. trios-bee-runner-1: about 6 of 48 +; turns ended after 0 steps. The bee for #9967 started at 14:50:47 on pool-2 key 9/10 and died at 14:50:54. +; The cause: a bee gets ONE key when it is dispatched (trios queen-dispatch.ts resolveWorkerProvider, +; round-robin) and keeps it. On 1305 the SDK retried the same key three times, the turn closed with no +; completion frame, and the dispatch ended "cause undetermined" while 33 other keys in two pools (24 and +; 10) stood by. The reviewer already does the right thing (trios queen-tick.ts, "a transient refusal +; tries the next lane too", after it lost 16 of 16 calls to 1302/1305 on 2026-09-23); bees did not. +; The rule here, after Envoy's retry_host_predicate previous_hosts plus outlier ejection, and the LiteLLM +; router's cooldown-then-fallback: +; 1. classify the refusal: transient, quota, auth or a bad request (error_kind); +; 2. a transient refusal rests the key for seconds (cooldown_seconds), a quota or auth refusal closes +; it for the long break -- 1302 and 1305 never close a key and never retire work; +; 3. the next attempt goes to the next free key of the pool, never the key that just failed +; (next_free); 1305 is an overload of the pool's MODEL, not of one key, so once POOL_OVERLOAD_KEYS +; distinct keys of one pool answered 1305 inside OVERLOAD_WINDOW_SECONDS the bee moves to the next +; pool instead of burning the rest (pool_overloaded); +; 4. at most MAX_ATTEMPTS_PER_TURN attempts, then the turn ends as provider_overloaded (trios#1708), +; its claim is released (trios#1886), and the end is not charged to the issue (charges_issue). +; Masks: bit i of a u32 mask is key i of the pool (or pool i of the pool list); a pool holds at most +; POOL_MAX keys. NO_KEY means "none left". The host keeps the clock and the masks; this file decides. +; Not here: the wiring into trios. It lands as generated code, not hand-written TypeScript. ASCII only (L3). +; phi^2 + 1/phi^2 = 3 | TRINITY + +module QueenKeyFailover; + +; --- 1. What the provider said ----------------------------------------------------------------------- + +/// KE_OK: the request was answered. KE_TRANSIENT: refused for now, another key or a moment later may +/// pass. KE_QUOTA: the key's quota, plan or balance is spent. KE_AUTH: the key itself is refused. +/// KE_REQUEST: the request is wrong; every key would answer the same. +pub const KE_OK : u8 = 0; +pub const KE_TRANSIENT : u8 = 1; +pub const KE_QUOTA : u8 = 2; +pub const KE_AUTH : u8 = 3; +pub const KE_REQUEST : u8 = 4; + +/// The z.ai rate limit and temporary overload codes. Both arrive as HTTP 429 and both clear on their own. +pub const ZAI_RATE_LIMIT : u32 = 1302; +pub const ZAI_OVERLOAD : u32 = 1305; + +/// The z.ai codes whose documented meaning is a spent quota, plan or balance: 1113 and 1308-1321 +/// except 1312, the closed list ZAI_QUOTA_EXHAUSTED_CODES in trios queen-dispatch.ts (#1301). +pub fn is_quota_code(code: u32) -> bool { + if (code == 1113) { return true; } + if (code == 1312) { return false; } + if (code < 1308) { return false; } + return code <= 1321; +} + +/// `status` is the HTTP status, 0 when no response came at all (timeout, reset, abort). `zai_code` is +/// the business code in the body, 0 when there is none. The code wins over the status, because z.ai +/// sends a spent quota and a passing overload with the same 429. +pub fn error_kind(status: u32, zai_code: u32) -> u8 { + if (is_quota_code(zai_code)) { return KE_QUOTA; } + if (zai_code == ZAI_RATE_LIMIT) { return KE_TRANSIENT; } + if (zai_code == ZAI_OVERLOAD) { return KE_TRANSIENT; } + if (status == 0) { return KE_TRANSIENT; } + if (status == 402) { return KE_QUOTA; } + if (status == 401) { return KE_AUTH; } + if (status == 403) { return KE_AUTH; } + if (status == 408) { return KE_TRANSIENT; } + if (status == 429) { return KE_TRANSIENT; } + if (status >= 600) { return KE_REQUEST; } + if (status >= 500) { return KE_TRANSIENT; } + if (status >= 400) { return KE_REQUEST; } + return KE_OK; +} + +; --- 2. How long a refused key rests ------------------------------------------------------------------ + +/// A transient refusal rests the key COOL_BASE_SECONDS, doubled per strike in a row, never more than +/// COOL_CAP_SECONDS: long enough that the next bee does not hit the same overloaded key at once, short +/// enough that 34 keys are never all resting through a one-minute burst like 14:08. +pub const COOL_BASE_SECONDS : u32 = 15; +pub const COOL_CAP_SECONDS : u32 = 240; +/// A spent quota or a refused key closes for the long break the runtime already uses (30 minutes). +pub const KEY_BREAK_SECONDS : u32 = 1800; + +/// `strikes` is the key's transient refusals in a row before this one. The cap is the base doubled +/// exactly four times (an invariant pins it), so the doubling meets the cap and never steps past it. +pub fn cooldown_seconds(kind: u8, strikes: u32) -> u32 { + if (kind == KE_QUOTA) { return KEY_BREAK_SECONDS; } + if (kind == KE_AUTH) { return KEY_BREAK_SECONDS; } + if (kind != KE_TRANSIENT) { return 0; } + var wait : u32 = COOL_BASE_SECONDS; + var n : u32 = 0; + while (n < strikes) { + if (wait >= COOL_CAP_SECONDS) { return COOL_CAP_SECONDS; } + wait = wait * 2; + n = n + 1; + } + return wait; +} + +/// Whether the refusal closes the key for the long break. A transient refusal never does. +pub fn key_closes(kind: u8) -> bool { + if (kind == KE_QUOTA) { return true; } + return kind == KE_AUTH; +} + +; --- 3. The next key, the next pool ------------------------------------------------------------------ + +pub const POOL_MAX : u32 = 32; +pub const NO_KEY : u32 = 255; + +pub fn bit(i: u32) -> u32 { + if (i >= POOL_MAX) { return 0; } + return 1 << i; +} + +pub fn is_set(mask: u32, i: u32) -> bool { + return (mask & bit(i)) != 0; +} + +/// The next index after `last`, wrapping, that is neither tried in this turn nor blocked (resting or +/// closed), and is never `last` itself: the scan visits the size - 1 other indices only. `last` out of +/// range (NO_KEY) visits all of them from index 0. NO_KEY when none is free. A list longer than +/// POOL_MAX does not fit a mask and is refused (NO_KEY), never read as its first 32. The same rule +/// picks a key inside a pool and a pool inside the pool list. +pub fn next_free(tried: u32, blocked: u32, size: u32, last: u32) -> u32 { + if (size > POOL_MAX) { return NO_KEY; } + var start : u32 = 0; + var count : u32 = size; + if (last < size) { + start = last + 1; + count = size - 1; + } + var k : u32 = 0; + while (k < count) { + var i : u32 = (start + k) % size; + if (is_set(tried | blocked, i) == false) { return i; } + k = k + 1; + } + return NO_KEY; +} + +/// 1305 from POOL_OVERLOAD_KEYS distinct keys of one pool inside OVERLOAD_WINDOW_SECONDS is the +/// pool's model overloaded. A pool smaller than that is overloaded when every key said it. +pub const POOL_OVERLOAD_KEYS : u32 = 3; +pub const OVERLOAD_WINDOW_SECONDS : u32 = 60; + +pub fn pool_overloaded(distinct_keys_1305: u32, pool_size: u32) -> bool { + if (distinct_keys_1305 >= POOL_OVERLOAD_KEYS) { return true; } + if (pool_size == 0) { return false; } + return distinct_keys_1305 >= pool_size; +} + +; --- 4. The step after a refusal --------------------------------------------------------------------- + +/// FA_DONE: answered, go on. FA_NEXT_KEY: retry on next_free's key. FA_NEXT_POOL: retry on the next +/// pool. FA_GIVE_UP: no key left; the turn ends as provider_overloaded. FA_FAIL: the request is wrong. +pub const FA_DONE : u8 = 0; +pub const FA_NEXT_KEY : u8 = 1; +pub const FA_NEXT_POOL : u8 = 2; +pub const FA_GIVE_UP : u8 = 3; +pub const FA_FAIL : u8 = 4; + +/// The bound on attempts in one turn, the failed one included: three keys of the first pool to see an +/// overload, three of the second, and two to spare. It keeps a turn from spinning through 34 keys. +pub const MAX_ATTEMPTS_PER_TURN : u32 = 8; + +/// `attempts` counts the attempts of this turn so far, this one included. `next_key` and `next_pool` +/// are next_free's answers for the current pool and for the pool list (NO_KEY when none). +pub fn step(kind: u8, attempts: u32, next_key: u32, distinct_keys_1305: u32, pool_size: u32, next_pool: u32) -> u8 { + if (kind == KE_OK) { return FA_DONE; } + if (kind >= KE_REQUEST) { return FA_FAIL; } + if (attempts >= MAX_ATTEMPTS_PER_TURN) { return FA_GIVE_UP; } + var pool_out : bool = false; + if (kind == KE_TRANSIENT) { pool_out = pool_overloaded(distinct_keys_1305, pool_size); } + if (pool_out == false) { + if (next_key != NO_KEY) { return FA_NEXT_KEY; } + } + if (next_pool != NO_KEY) { return FA_NEXT_POOL; } + return FA_GIVE_UP; +} + +/// Whether a turn that ended on this kind counts against the issue. A provider that is overloaded, +/// spent or refusing a key says nothing about the work, so it never retires the issue; a wrong request +/// does. Only KE_REQUEST charges. This narrows dispatch_exit.t27 counts_against_issue, which charges +/// every DX_PROVIDER_ERROR. +pub fn charges_issue(kind: u8) -> bool { + return kind == KE_REQUEST; +} + +; --- 5. Tests ---------------------------------------------------------------------------------------- + +test the_quota_list_is_the_closed_list_of_the_runtime { + assert is_quota_code(1113) == true; + assert is_quota_code(1308) == true; + assert is_quota_code(1311) == true; + assert is_quota_code(1312) == false; + assert is_quota_code(1313) == true; + assert is_quota_code(1321) == true; + assert is_quota_code(1322) == false; + assert is_quota_code(1307) == false; + assert is_quota_code(1112) == false; + assert is_quota_code(1114) == false; + assert is_quota_code(ZAI_RATE_LIMIT) == false; + assert is_quota_code(ZAI_OVERLOAD) == false; + assert is_quota_code(0) == false; +} + +test the_code_wins_over_the_status { + assert error_kind(429, 1305) == KE_TRANSIENT; + assert error_kind(429, 1302) == KE_TRANSIENT; + assert error_kind(429, 1113) == KE_QUOTA; + assert error_kind(429, 1316) == KE_QUOTA; + assert error_kind(200, 1308) == KE_QUOTA; + assert error_kind(400, 1305) == KE_TRANSIENT; + assert error_kind(400, 1302) == KE_TRANSIENT; + assert error_kind(429, 1312) == KE_TRANSIENT; +} + +test a_status_without_a_code_falls_in_its_kind { + assert error_kind(0, 0) == KE_TRANSIENT; + assert error_kind(200, 0) == KE_OK; + assert error_kind(399, 0) == KE_OK; + assert error_kind(400, 0) == KE_REQUEST; + assert error_kind(401, 0) == KE_AUTH; + assert error_kind(402, 0) == KE_QUOTA; + assert error_kind(403, 0) == KE_AUTH; + assert error_kind(404, 0) == KE_REQUEST; + assert error_kind(407, 0) == KE_REQUEST; + assert error_kind(408, 0) == KE_TRANSIENT; + assert error_kind(409, 0) == KE_REQUEST; + assert error_kind(428, 0) == KE_REQUEST; + assert error_kind(429, 0) == KE_TRANSIENT; + assert error_kind(430, 0) == KE_REQUEST; + assert error_kind(499, 0) == KE_REQUEST; + assert error_kind(500, 0) == KE_TRANSIENT; + assert error_kind(503, 0) == KE_TRANSIENT; + assert error_kind(529, 0) == KE_TRANSIENT; + assert error_kind(599, 0) == KE_TRANSIENT; + assert error_kind(600, 0) == KE_REQUEST; +} + +test a_transient_refusal_rests_seconds_and_a_spent_key_closes { + assert cooldown_seconds(KE_TRANSIENT, 0) == 15; + assert cooldown_seconds(KE_TRANSIENT, 1) == 30; + assert cooldown_seconds(KE_TRANSIENT, 2) == 60; + assert cooldown_seconds(KE_TRANSIENT, 3) == 120; + assert cooldown_seconds(KE_TRANSIENT, 4) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_TRANSIENT, 5) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_TRANSIENT, 4000000000) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_QUOTA, 0) == KEY_BREAK_SECONDS; + assert cooldown_seconds(KE_AUTH, 7) == KEY_BREAK_SECONDS; + assert cooldown_seconds(KE_OK, 3) == 0; + assert cooldown_seconds(KE_REQUEST, 3) == 0; + assert key_closes(KE_QUOTA) == true; + assert key_closes(KE_AUTH) == true; + assert key_closes(KE_TRANSIENT) == false; + assert key_closes(KE_OK) == false; + assert key_closes(KE_REQUEST) == false; +} + +test bits_name_keys { + assert bit(0) == 1; + assert bit(3) == 8; + assert bit(31) == 2147483648; + assert bit(32) == 0; + assert is_set(8, 3) == true; + assert is_set(8, 2) == false; + assert is_set(4294967295, 31) == true; + assert is_set(4294967295, 32) == false; +} + +test the_next_key_is_never_the_one_that_failed { + // the #9967 case: pool 2 has 10 keys, key 9 answered 1305; the next try wraps to key 0 + assert next_free(bit(9), 0, 10, 9) == 0; + // even when the failed key is not marked tried, it is skipped + assert next_free(0, 0, 10, 9) == 0; + assert next_free(0, 0, 10, 4) == 5; + // a pool of one key has no other key + assert next_free(0, 0, 1, 0) == NO_KEY; + // tried and resting keys are both skipped + assert next_free(bit(5), bit(6), 10, 4) == 7; + assert next_free(bit(0) | bit(1), 0, 3, 2) == NO_KEY; + // a fresh turn starts at key 0 + assert next_free(0, 0, 24, NO_KEY) == 0; + assert next_free(bit(0), 0, 24, NO_KEY) == 1; + // every key but the failed one is taken + assert next_free(4294967295 - bit(9), 0, 10, 9) == NO_KEY; + // an empty pool has nothing + assert next_free(0, 0, 0, NO_KEY) == NO_KEY; + // `last` just past the end is out of range: the scan starts at 0 + assert next_free(0, 0, 10, 10) == 0; + // a full mask of POOL_MAX keys wraps from the top index + assert next_free(0, 0, 32, 31) == 0; + assert next_free(0, 0, 32, NO_KEY) == 0; + assert next_free(0, 4294967295 - bit(31), 32, 30) == 31; + // a list longer than POOL_MAX does not fit a mask and is refused + assert next_free(0, 0, 33, NO_KEY) == NO_KEY; + assert next_free(0, 0, 40, 31) == NO_KEY; +} + +test an_overload_seen_on_three_keys_is_the_pools_model { + assert pool_overloaded(0, 24) == false; + assert pool_overloaded(2, 24) == false; + assert pool_overloaded(3, 24) == true; + assert pool_overloaded(16, 24) == true; + assert pool_overloaded(2, 2) == true; + assert pool_overloaded(1, 2) == false; + assert pool_overloaded(1, 1) == true; + assert pool_overloaded(0, 0) == false; +} + +/// The conformance table: one assert per row. Columns: kind, attempts, next key, distinct keys that +/// answered 1305 in the window, pool size, next pool. +test conformance_rows { + // r01 answered: go on + assert step(KE_OK, 1, 5, 0, 24, 1) == FA_DONE; + // r02 a wrong request fails, whatever keys are left + assert step(KE_REQUEST, 1, 5, 0, 24, 1) == FA_FAIL; + // r03 an unknown kind is read as a wrong request + assert step(9, 1, 5, 0, 24, 1) == FA_FAIL; + // r04 the #9967 case: first 1305 on pool 2, keys left: next key, not the same key + assert step(KE_TRANSIENT, 1, 0, 1, 10, 0) == FA_NEXT_KEY; + // r05 a rate limit (1302) on a fresh pool: next key + assert step(KE_TRANSIENT, 1, 3, 0, 24, 1) == FA_NEXT_KEY; + // r06 the third distinct key said 1305: the model is overloaded, next pool + assert step(KE_TRANSIENT, 3, 4, 3, 24, 1) == FA_NEXT_POOL; + // r07 the 14:08 burst: overloaded and no other pool: give up, do not burn the rest + assert step(KE_TRANSIENT, 3, 4, 3, 24, NO_KEY) == FA_GIVE_UP; + // r08 a spent quota on one key does not mean an overloaded pool: next key + assert step(KE_QUOTA, 1, 2, 5, 24, 1) == FA_NEXT_KEY; + // r09 a refused key: next key + assert step(KE_AUTH, 1, 2, 0, 24, 1) == FA_NEXT_KEY; + // r10 no key left in the pool: next pool + assert step(KE_TRANSIENT, 2, NO_KEY, 1, 24, 1) == FA_NEXT_POOL; + // r11 no key and no pool left: give up + assert step(KE_QUOTA, 2, NO_KEY, 0, 24, NO_KEY) == FA_GIVE_UP; + // r12 the attempt bound stops a turn with keys still left + assert step(KE_TRANSIENT, MAX_ATTEMPTS_PER_TURN, 4, 0, 24, 1) == FA_GIVE_UP; + // r13 one attempt below the bound still moves on + assert step(KE_TRANSIENT, MAX_ATTEMPTS_PER_TURN - 1, 4, 0, 24, 1) == FA_NEXT_KEY; + // r14 a two-key pool where both said 1305 is overloaded + assert step(KE_TRANSIENT, 2, NO_KEY, 2, 2, 0) == FA_NEXT_POOL; + // r15 a 503 with no 1305 seen keeps to the pool + assert step(KE_TRANSIENT, 2, 7, 0, 24, 1) == FA_NEXT_KEY; + // r16 the bound beats a wrong request? no: a wrong request fails first + assert step(KE_REQUEST, MAX_ATTEMPTS_PER_TURN, 4, 0, 24, 1) == FA_FAIL; + // r17 an answer after many attempts is still an answer + assert step(KE_OK, MAX_ATTEMPTS_PER_TURN, NO_KEY, 9, 24, NO_KEY) == FA_DONE; +} + +test an_overloaded_provider_never_retires_the_issue { + assert charges_issue(KE_TRANSIENT) == false; + assert charges_issue(KE_QUOTA) == false; + assert charges_issue(KE_AUTH) == false; + assert charges_issue(KE_REQUEST) == true; + assert charges_issue(KE_OK) == false; + assert charges_issue(9) == false; +} + +; --- 6. Invariants ----------------------------------------------------------------------------------- + +invariant the_kinds_and_steps_are_distinct { + assert KE_OK == 0 and KE_TRANSIENT == 1 and KE_QUOTA == 2 and KE_AUTH == 3 and KE_REQUEST == 4; + assert FA_DONE == 0 and FA_NEXT_KEY == 1 and FA_NEXT_POOL == 2 and FA_GIVE_UP == 3 and FA_FAIL == 4; +} + +/// A transient rest is always shorter than the long break, so a 1305 can never close a key. +invariant a_rest_is_shorter_than_a_break { + assert COOL_BASE_SECONDS > 0 and COOL_CAP_SECONDS == COOL_BASE_SECONDS * 16; + assert COOL_CAP_SECONDS < KEY_BREAK_SECONDS and KEY_BREAK_SECONDS == 1800; +} + +/// The bound leaves room to see an overload in one pool and still try the next: the overload is +/// detected before the bound stops the turn. +invariant the_bound_leaves_room_for_a_second_pool { + assert MAX_ATTEMPTS_PER_TURN > POOL_OVERLOAD_KEYS * 2; + assert POOL_OVERLOAD_KEYS >= 2 and OVERLOAD_WINDOW_SECONDS > 0; +} + +/// NO_KEY is no index a mask can hold. +invariant no_key_is_out_of_range { + assert POOL_MAX == 32 and NO_KEY >= POOL_MAX; + assert ZAI_RATE_LIMIT == 1302 and ZAI_OVERLOAD == 1305; +} diff --git a/trios/agent-server/specs/queen/key_failover.wasm b/trios/agent-server/specs/queen/key_failover.wasm new file mode 100755 index 0000000000..c8b33bc372 Binary files /dev/null and b/trios/agent-server/specs/queen/key_failover.wasm differ