diff --git a/trios/agent-server/apps/server/src/api/routes/queen-lease.ts b/trios/agent-server/apps/server/src/api/routes/queen-lease.ts index 0b321fa55b..fc77d9d6fe 100644 --- a/trios/agent-server/apps/server/src/api/routes/queen-lease.ts +++ b/trios/agent-server/apps/server/src/api/routes/queen-lease.ts @@ -15,16 +15,20 @@ * transport they have. One lease, one name, two kinds of client. */ +import { createHash, timingSafeEqual } from 'node:crypto' import { Hono } from 'hono' import type { Pool } from 'pg' import { createQueenPool } from '../../lib/db/queen-pool' +import { leased, vault, zaiLeaseCall } from '../services/queen-free-lane' +import * as KV from '../services/queen-key-vault-card.gen' import { acquireQueenLease, queenLeaseDatabaseUrl, queenLeaseStatus, releaseQueenLease, } from '../services/queen-lease' -import { runQueenTickOnce } from '../services/queen-tick' +import { zaiLeaseLanes } from '../services/queen-reviewer' +import { runningKeys, runQueenTickOnce } from '../services/queen-tick' const LEASE_NAME = 'queen-tick' @@ -174,3 +178,38 @@ export function createQueenLeaseRoute() { }) ) } + +/** What a lease forwards of the caller's request: never its token. */ +const ZAI_HEADERS = { + 'content-type': 'application/json', + 'anthropic-version': '2023-06-01', +} +/** The Mac reviewer's z.ai calls, made here so no key leaves the Queen. */ +export function createQueenZaiLeaseRoute() { + return new Hono().all('/*', async (c) => { + const token = process.env.TRIOS_QUEEN_REVIEW_LEASE_TOKEN ?? '' + const sha = (s: string) => createHash('sha256').update(s).digest() + if (token.length < 32) return c.json({ error: 'lease_off' }, 503) + const given = sha(c.req.header('authorization') ?? '') + if (!timingSafeEqual(given, sha(`Bearer ${token}`))) + return c.json({ error: 'forbidden' }, 403) + const url = queenLeaseDatabaseUrl() + const taken = url ? await runningKeys(poolForTick(url)) : [] + const lanes = zaiLeaseLanes(taken, leased) + const total = zaiLeaseLanes([], leased).length + leased.size + const n = leased.size + const ask = [KV.LEASE_REVIEW, KV.SRC_STORE, n, lanes.length, total, n, 1] + const answer = vault('lease_answer', ...ask) + const path = c.req.url.replace(/^.*?\/zai-lease/, '') + const body = c.req.method === 'GET' ? undefined : await c.req.arrayBuffer() + const init = { method: c.req.method, headers: ZAI_HEADERS, body } + const res = + answer === KV.LA_GRANT && (await zaiLeaseCall(lanes, path, init)) + if (res) return res + const refused = answer === KV.LA_REFUSE + const wait = vault('retry_after_seconds', refused ? answer : KV.LA_WAIT) + c.header('retry-after', String(wait)) + const error = { type: 'rate_limit_error', message: 'no z.ai lane free' } + return c.json({ type: 'error', error }, refused ? 403 : 429) + }) +} diff --git a/trios/agent-server/apps/server/src/api/server.ts b/trios/agent-server/apps/server/src/api/server.ts index b27064e910..f6919a5965 100644 --- a/trios/agent-server/apps/server/src/api/server.ts +++ b/trios/agent-server/apps/server/src/api/server.ts @@ -57,7 +57,10 @@ import { createQueenKanbanRoute, createQueenPublicBoardRoute, } from './routes/queen-kanban' -import { createQueenLeaseRoute } from './routes/queen-lease' +import { + createQueenLeaseRoute, + createQueenZaiLeaseRoute, +} from './routes/queen-lease' import { createQueenNeedsYouRoute } from './routes/queen-needs-you' import { createQueenPublicActivityRoute } from './routes/queen-public-activity' import { createQueenPublicActorsRoute } from './routes/queen-public-actors' @@ -502,6 +505,8 @@ export async function createHttpServer(config: HttpServerConfig) { .route('/queen/public-earnings', createQueenPublicEarningsRoute()) // Separate server-to-server capability; never a public-read or operator-token route. .route('/queen/contributor-keys', createQueenContributorKeysRoute()) + // The Mac reviewer's z.ai lease (specs/queen/key_vault.t27): its own token. + .route('/queen/zai-lease', createQueenZaiLeaseRoute()) // The vault (gHashTag/trios#1759): every request is Ed25519-signed and // judged by specs/vault/policy.t27 request_code; no browser Origin, so no // trusted-origin guard. 503 for everything unless TRIOS_VAULT=on. diff --git a/trios/agent-server/apps/server/src/api/services/queen-free-lane.ts b/trios/agent-server/apps/server/src/api/services/queen-free-lane.ts index bf655416b2..98764c514b 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-free-lane.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-free-lane.ts @@ -21,7 +21,9 @@ import type { Pool } from 'pg' import type { AppLlm } from './queen-app' +import { loadCardWasm } from './queen-card-wasm' import { reviewLaneCandidates, type WorkerProvider } from './queen-dispatch' +import * as KV from './queen-key-vault-card.gen' import { chooseReviewerLane, defaultReviewerLlm, @@ -72,3 +74,41 @@ export function freeLaneLlm( return { ok: false, error: last, transient: true } } } + +/** Lease decisions are key_vault.t27's; `leased` is the z.ai keys out now. */ +export const vault = (fn: string, ...args: number[]) => + loadCardWasm('queen/key_vault.wasm').call(fn, ...args) +export const leased = new Map() + +/** One leased request: a lane makes it, a refused lane hands it on. */ +export async function zaiLeaseCall( + lanes: WorkerProvider[], + path: string, + init: RequestInit & { headers: Record }, +) { + let last: Response | null = null + for (const [i, lane] of lanes.slice(0, KV.LEASE_TRIES).entries()) { + const key = reviewerLaneKey(lane) + const id = Symbol() + const free = () => leased.get(key) === id && leased.delete(key) + leased.set(key, id) + setTimeout(free, KV.LEASE_SECONDS * 1000) + const res = await fetch(`https://api.z.ai/api/anthropic${path}`, { + ...init, + headers: { ...init.headers, authorization: `Bearer ${lane.apiKey}` }, + signal: AbortSignal.timeout(KV.LEASE_SECONDS * 1000), + }).catch(() => null) + const headers = { 'content-type': res?.headers.get('content-type') ?? '' } + if (res?.ok && res.body) { + const done = new TransformStream({ flush: () => void free() }) + return new Response(res.body.pipeThrough(done), { headers }) + } + free() + const text = (await res?.text().catch(() => '')) ?? '' + const code = Number(/"code"\s*:\s*"?(\d+)/.exec(text)?.[1] ?? 0) + last = res && new Response(text, { status: res.status, headers }) + if (vault('lease_step', res?.status ?? 0, code, i + 1) === KV.LS_ANSWER) + break + } + return last +} diff --git a/trios/agent-server/apps/server/src/api/services/queen-key-vault-card.gen.ts b/trios/agent-server/apps/server/src/api/services/queen-key-vault-card.gen.ts new file mode 100644 index 0000000000..69fac0c408 --- /dev/null +++ b/trios/agent-server/apps/server/src/api/services/queen-key-vault-card.gen.ts @@ -0,0 +1,70 @@ +// Generated by `t27c gen-ts` from key_vault.t27. Do not edit. +// +// Every name, value and type here comes from that spec. Change the spec and +// regenerate; an edit made here is lost the next time anyone builds. + +export const SRC_STORE = 0 satisfies number; +export const SRC_PRIVATE = 1 satisfies number; +// t27c gen-ts: fn may_serve was not emitted -- this backend lowers declarations, not bodies. +export const START_LEASE = 0 satisfies number; +export const START_OWN_KEYS = 1 satisfies number; +export const START_REFUSE = 2 satisfies number; +// t27c gen-ts: fn consumer_start was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn start_warns was not emitted -- this backend lowers declarations, not bodies. +export const LEASE_REVIEW = 0 satisfies number; +export const LEASE_WORKER = 1 satisfies number; +export const REVIEW_FLOOR_LANES = 2 satisfies number; +export const REVIEW_HOLDERS = 3 satisfies number; +// t27c gen-ts: fn lanes_of was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn review_share was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn worker_share was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn holder_cap was not emitted -- this backend lowers declarations, not bodies. +export const LA_GRANT = 0 satisfies number; +export const LA_WAIT = 1 satisfies number; +export const LA_REFUSE = 2 satisfies number; +// t27c gen-ts: fn lease_answer was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn retry_after_seconds was not emitted -- this backend lowers declarations, not bodies. +export const LEASE_SECONDS = 600 satisfies number; +export const LEASE_TRIES = 3 satisfies number; +export const LS_ANSWER = 0 satisfies number; +export const LS_NEXT_LANE = 1 satisfies number; +// t27c gen-ts: fn lease_step was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a TestBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +// t27c gen-ts: a InvariantBlock was not emitted -- it is checked by the compiler, not by the artifact. +export const COOL_BASE_SECONDS = 15 satisfies number; +export const KE_AUTH = 3 satisfies number; +export const KE_OK = 0 satisfies number; +export const KE_QUOTA = 2 satisfies number; +export const KE_REQUEST = 4 satisfies number; +export const KE_TRANSIENT = 1 satisfies number; +export const ZAI_OVERLOAD = 1305 satisfies number; +export const ZAI_RATE_LIMIT = 1302 satisfies number; +// t27c gen-ts: fn error_kind was not emitted -- this backend lowers declarations, not bodies. +// t27c gen-ts: fn is_quota_code was not emitted -- this backend lowers declarations, not bodies. +export const U32_MAX = 4294967295 satisfies number; +export const ZAI_REQUESTS_PER_KEY = 2 satisfies number; + +// Declaration order, which the spec's own laws depend on. +export const __STRUCT_ORDER__ = [] as const; +export const __DECL_ORDER__ = ["SRC_STORE", "SRC_PRIVATE", "START_LEASE", "START_OWN_KEYS", "START_REFUSE", "LEASE_REVIEW", "LEASE_WORKER", "REVIEW_FLOOR_LANES", "REVIEW_HOLDERS", "LA_GRANT", "LA_WAIT", "LA_REFUSE", "LEASE_SECONDS", "LEASE_TRIES", "LS_ANSWER", "LS_NEXT_LANE", "COOL_BASE_SECONDS", "KE_AUTH", "KE_OK", "KE_QUOTA", "KE_REQUEST", "KE_TRANSIENT", "ZAI_OVERLOAD", "ZAI_RATE_LIMIT", "U32_MAX", "ZAI_REQUESTS_PER_KEY"] as const; + +// What this spec holds and this backend did not print. Empty is the whole +// story most of the time; an entry here is a promise the artifact does not +// keep, and reading it is how a tool tells a partial module from a complete +// one without parsing comments. +// +// A LIST, not a map keyed by name. `specs/numeric/formats.t27` declares four +// separate consts called `result`, one per test block, and under an object +// literal three of the four omissions vanished into the fourth -- a record of +// what went missing that itself went missing. A spec is free to reuse a +// name; this file is not free to lose the second one. +export const __NOT_EMITTED__ = Object.freeze([] as const); diff --git a/trios/agent-server/apps/server/src/api/services/queen-reviewer.ts b/trios/agent-server/apps/server/src/api/services/queen-reviewer.ts index 097bc573e1..c90f1f72c1 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-reviewer.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-reviewer.ts @@ -356,6 +356,16 @@ export function reviewerLaneBackedOff( return true } +/** z.ai review lanes no bee, backoff or lease holds (key_vault.t27). */ +export const zaiLeaseLanes = (taken: number[], busy: Map) => + reviewLaneCandidates(taken).filter( + (lane) => + !!lane.apiKey && + !!lane.baseUrl?.includes('api.z.ai') && + !reviewerLaneBackedOff(lane) && + !busy.has(reviewerLaneKey(lane)), + ) + /** For suites: every lane is trusted again. */ export function forgetReviewerLaneFailures(): void { failedLanes.clear() diff --git a/trios/agent-server/apps/server/tests/api/routes/route-guard.test.ts b/trios/agent-server/apps/server/tests/api/routes/route-guard.test.ts index 86447c0769..80ce623124 100644 --- a/trios/agent-server/apps/server/tests/api/routes/route-guard.test.ts +++ b/trios/agent-server/apps/server/tests/api/routes/route-guard.test.ts @@ -57,6 +57,8 @@ const report = auditServer(source, DEFAULT_ALLOWLIST) // RE-MEASURED 2026-10-11, queen merged into vault-mvp (trios#1759): both // joined, so the list is fourteen, recomputed with --no-allowlist on the // merged server.ts. +// RE-MEASURED 2026-10-19: /queen/zai-lease joined (key_vault.t27), so fifteen. +// The Mac reviewer calls it with no Origin; its guard is its own lease token. const EXPECTED_UNGUARDED_WITHOUT_ALLOWLIST = [ '/api/inngest', '/hosting', @@ -71,6 +73,7 @@ const EXPECTED_UNGUARDED_WITHOUT_ALLOWLIST = [ '/queen/roadmap', '/queen/runner', '/queen/tree', + '/queen/zai-lease', '/vault', ] @@ -199,7 +202,9 @@ describe('route-guard audit over src/api/server.ts', () => { // tools/route-guard-audit.mjs, not copied. 66: /vault, /hosting and // /queen/public-net-credits together; 18 prefix guards, 20 guarded // sub-apps, 18 public-read. - expect(report.totalMounts).toBe(66) + // RE-MEASURED 2026-10-19, the key vault lease (key_vault.t27): 66 became 67 + // with /queen/zai-lease, allowlisted with its own-token reason. + expect(report.totalMounts).toBe(67) expect(report.prefixGuardCount).toBe(18) expect(report.guardedSubAppCount).toBe(20) expect(report.publicReadCount).toBe(18) @@ -212,7 +217,7 @@ describe('route-guard audit over src/api/server.ts', () => { expect(report.entriesMissingReason).toEqual([]) }) - it('reports exactly the fourteen reasoned exceptions when the allowlist is dropped', () => { + it('reports exactly the fifteen reasoned exceptions when the allowlist is dropped', () => { // The classifier reports mounts in file order; the assertion is on the // exact set, so both sides are sorted before comparing. expect([...unguardedMounts(source, [])].sort()).toEqual( @@ -289,7 +294,9 @@ describe('route-guard audit over src/api/server.ts', () => { // RE-MEASURED 2026-10-10, queen merged in: still forty, recomputed. // RE-MEASURED 2026-10-10, the network job: forty became forty-one. The // eighteenth public-read is /queen/public-net-credits. - expect(queenMounts.length).toBe(41) + // RE-MEASURED 2026-10-19: forty-one became forty-two. The eleventh + // allowlisted mount is /queen/zai-lease, behind its own lease token. + expect(queenMounts.length).toBe(42) const counts: Record = { 'public-read': 0, @@ -306,7 +313,7 @@ describe('route-guard audit over src/api/server.ts', () => { 'public-read': 18, 'prefix-guard': 0, wrapper: 13, - unguarded: 10, + unguarded: 11, }) // Every unguarded /queen mount must be one of the allowlisted shells. diff --git a/trios/agent-server/specs/PIN b/trios/agent-server/specs/PIN index 8a3a8c610b..038cd08ea2 100644 --- a/trios/agent-server/specs/PIN +++ b/trios/agent-server/specs/PIN @@ -130,3 +130,7 @@ claim: queen/claim.wasm sha256 08fc2b32d46742638c00d20fe1e64dd81289080dcfc8fa662 review_first card: gHashTag/t27 specs/queen/review_first.t27 (t27#10555, PR t27#10562), byte-identical; review_first.wasm = t27c gen-c + zig 0.16.0 cc --target=wasm32-wasi -Oz -DNDEBUG -mexec-model=reactor, exports every fn of the card: kind_of,in_review_queue,backlog_open,admission,admits,round_refused (reproducible: two builds, one hash); constants: t27c gen-ts > apps/server/src/api/services/queen-review-first-card.gen.ts review_first: queen/review_first.t27 sha256 c0ec4028feccb9740ddd434f8a1ee8f4fce9b7a4ce2dd1664de504a0b5d06e88 review_first: queen/review_first.wasm sha256 6c9fbeca0bc9dc98122ee4dcc8cdf1ad0c37c659625422828281309875555f2f +key_vault card: gHashTag/t27 specs/queen/key_vault.t27 (t27#10564, PR t27#10567 head ae8bcbe6f, blob 341c272cb4b76382eecfe81ae901bf750458bba1), byte-identical; it imports error_kind and its codes from queen/key_failover.t27 (blob 3ce96bef2b149e03cfd5879491e448b8c8c5a9a8, vendored byte-identical, the same file trios#2045 vendors) and ZAI_REQUESTS_PER_KEY from queen/reviewer_sizing.t27 (already vendored), so gen-c compiles them in; key_vault.wasm = t27c 0.5.2 gen-c + zig 0.16.0 cc --target=wasm32-wasi -Oz -DNDEBUG -mexec-model=reactor, exports may_serve,consumer_start,start_warns,lanes_of,review_share,worker_share,holder_cap,lease_answer,retry_after_seconds,lease_step,error_kind (reproducible: two builds, one hash, 797 bytes); constants: t27c gen-ts > apps/server/src/api/services/queen-key-vault-card.gen.ts (sha256 666f3ae917f810d6be10cf1d8bec0e140a277f330201276f37d1cef290c3d341) +key_vault: queen/key_vault.t27 sha256 caa5c0913648ec6eb545c0544fe0d51e383c42f5b4617ae25db6f542f2fa4097 +key_vault: queen/key_failover.t27 sha256 822367357fc6b487c1c7fb7ac8369165e6a7e6636fd9b0dcafec29e9a6274301 +key_vault: queen/key_vault.wasm sha256 8eb18f8eb0d6ff7866c51b8771aeff69dae25db71254b557401332154e7bf20f diff --git a/trios/agent-server/specs/queen/key_failover.t27 b/trios/agent-server/specs/queen/key_failover.t27 new file mode 100644 index 0000000000..3ce96bef2b --- /dev/null +++ b/trios/agent-server/specs/queen/key_failover.t27 @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/queen/key_failover.t27 -- a bee whose key is refused moves to another key, then another pool (gHashTag/t27#9993). +; Measured 2026-10-17 13:00-15:18Z in the Railway logs of trios-bee-runner-2: 93 bee turns, 17 of them +; ended after 0 steps; at least 22 AI_RetryError on z.ai "429 code 1305" (temporary overload), in bursts +; at 13:53, 13:59, 14:08 (16 in one minute), 14:19, 14:47 and 14:50. trios-bee-runner-1: about 6 of 48 +; turns ended after 0 steps. The bee for #9967 started at 14:50:47 on pool-2 key 9/10 and died at 14:50:54. +; The cause: a bee gets ONE key when it is dispatched (trios queen-dispatch.ts resolveWorkerProvider, +; round-robin) and keeps it. On 1305 the SDK retried the same key three times, the turn closed with no +; completion frame, and the dispatch ended "cause undetermined" while 33 other keys in two pools (24 and +; 10) stood by. The reviewer already does the right thing (trios queen-tick.ts, "a transient refusal +; tries the next lane too", after it lost 16 of 16 calls to 1302/1305 on 2026-09-23); bees did not. +; The rule here, after Envoy's retry_host_predicate previous_hosts plus outlier ejection, and the LiteLLM +; router's cooldown-then-fallback: +; 1. classify the refusal: transient, quota, auth or a bad request (error_kind); +; 2. a transient refusal rests the key for seconds (cooldown_seconds), a quota or auth refusal closes +; it for the long break -- 1302 and 1305 never close a key and never retire work; +; 3. the next attempt goes to the next free key of the pool, never the key that just failed +; (next_free); 1305 is an overload of the pool's MODEL, not of one key, so once POOL_OVERLOAD_KEYS +; distinct keys of one pool answered 1305 inside OVERLOAD_WINDOW_SECONDS the bee moves to the next +; pool instead of burning the rest (pool_overloaded); +; 4. at most MAX_ATTEMPTS_PER_TURN attempts, then the turn ends as provider_overloaded (trios#1708), +; its claim is released (trios#1886), and the end is not charged to the issue (charges_issue). +; Masks: bit i of a u32 mask is key i of the pool (or pool i of the pool list); a pool holds at most +; POOL_MAX keys. NO_KEY means "none left". The host keeps the clock and the masks; this file decides. +; Not here: the wiring into trios. It lands as generated code, not hand-written TypeScript. ASCII only (L3). +; phi^2 + 1/phi^2 = 3 | TRINITY + +module QueenKeyFailover; + +; --- 1. What the provider said ----------------------------------------------------------------------- + +/// KE_OK: the request was answered. KE_TRANSIENT: refused for now, another key or a moment later may +/// pass. KE_QUOTA: the key's quota, plan or balance is spent. KE_AUTH: the key itself is refused. +/// KE_REQUEST: the request is wrong; every key would answer the same. +pub const KE_OK : u8 = 0; +pub const KE_TRANSIENT : u8 = 1; +pub const KE_QUOTA : u8 = 2; +pub const KE_AUTH : u8 = 3; +pub const KE_REQUEST : u8 = 4; + +/// The z.ai rate limit and temporary overload codes. Both arrive as HTTP 429 and both clear on their own. +pub const ZAI_RATE_LIMIT : u32 = 1302; +pub const ZAI_OVERLOAD : u32 = 1305; + +/// The z.ai codes whose documented meaning is a spent quota, plan or balance: 1113 and 1308-1321 +/// except 1312, the closed list ZAI_QUOTA_EXHAUSTED_CODES in trios queen-dispatch.ts (#1301). +pub fn is_quota_code(code: u32) -> bool { + if (code == 1113) { return true; } + if (code == 1312) { return false; } + if (code < 1308) { return false; } + return code <= 1321; +} + +/// `status` is the HTTP status, 0 when no response came at all (timeout, reset, abort). `zai_code` is +/// the business code in the body, 0 when there is none. The code wins over the status, because z.ai +/// sends a spent quota and a passing overload with the same 429. +pub fn error_kind(status: u32, zai_code: u32) -> u8 { + if (is_quota_code(zai_code)) { return KE_QUOTA; } + if (zai_code == ZAI_RATE_LIMIT) { return KE_TRANSIENT; } + if (zai_code == ZAI_OVERLOAD) { return KE_TRANSIENT; } + if (status == 0) { return KE_TRANSIENT; } + if (status == 402) { return KE_QUOTA; } + if (status == 401) { return KE_AUTH; } + if (status == 403) { return KE_AUTH; } + if (status == 408) { return KE_TRANSIENT; } + if (status == 429) { return KE_TRANSIENT; } + if (status >= 600) { return KE_REQUEST; } + if (status >= 500) { return KE_TRANSIENT; } + if (status >= 400) { return KE_REQUEST; } + return KE_OK; +} + +; --- 2. How long a refused key rests ------------------------------------------------------------------ + +/// A transient refusal rests the key COOL_BASE_SECONDS, doubled per strike in a row, never more than +/// COOL_CAP_SECONDS: long enough that the next bee does not hit the same overloaded key at once, short +/// enough that 34 keys are never all resting through a one-minute burst like 14:08. +pub const COOL_BASE_SECONDS : u32 = 15; +pub const COOL_CAP_SECONDS : u32 = 240; +/// A spent quota or a refused key closes for the long break the runtime already uses (30 minutes). +pub const KEY_BREAK_SECONDS : u32 = 1800; + +/// `strikes` is the key's transient refusals in a row before this one. The cap is the base doubled +/// exactly four times (an invariant pins it), so the doubling meets the cap and never steps past it. +pub fn cooldown_seconds(kind: u8, strikes: u32) -> u32 { + if (kind == KE_QUOTA) { return KEY_BREAK_SECONDS; } + if (kind == KE_AUTH) { return KEY_BREAK_SECONDS; } + if (kind != KE_TRANSIENT) { return 0; } + var wait : u32 = COOL_BASE_SECONDS; + var n : u32 = 0; + while (n < strikes) { + if (wait >= COOL_CAP_SECONDS) { return COOL_CAP_SECONDS; } + wait = wait * 2; + n = n + 1; + } + return wait; +} + +/// Whether the refusal closes the key for the long break. A transient refusal never does. +pub fn key_closes(kind: u8) -> bool { + if (kind == KE_QUOTA) { return true; } + return kind == KE_AUTH; +} + +; --- 3. The next key, the next pool ------------------------------------------------------------------ + +pub const POOL_MAX : u32 = 32; +pub const NO_KEY : u32 = 255; + +pub fn bit(i: u32) -> u32 { + if (i >= POOL_MAX) { return 0; } + return 1 << i; +} + +pub fn is_set(mask: u32, i: u32) -> bool { + return (mask & bit(i)) != 0; +} + +/// The next index after `last`, wrapping, that is neither tried in this turn nor blocked (resting or +/// closed), and is never `last` itself: the scan visits the size - 1 other indices only. `last` out of +/// range (NO_KEY) visits all of them from index 0. NO_KEY when none is free. A list longer than +/// POOL_MAX does not fit a mask and is refused (NO_KEY), never read as its first 32. The same rule +/// picks a key inside a pool and a pool inside the pool list. +pub fn next_free(tried: u32, blocked: u32, size: u32, last: u32) -> u32 { + if (size > POOL_MAX) { return NO_KEY; } + var start : u32 = 0; + var count : u32 = size; + if (last < size) { + start = last + 1; + count = size - 1; + } + var k : u32 = 0; + while (k < count) { + var i : u32 = (start + k) % size; + if (is_set(tried | blocked, i) == false) { return i; } + k = k + 1; + } + return NO_KEY; +} + +/// 1305 from POOL_OVERLOAD_KEYS distinct keys of one pool inside OVERLOAD_WINDOW_SECONDS is the +/// pool's model overloaded. A pool smaller than that is overloaded when every key said it. +pub const POOL_OVERLOAD_KEYS : u32 = 3; +pub const OVERLOAD_WINDOW_SECONDS : u32 = 60; + +pub fn pool_overloaded(distinct_keys_1305: u32, pool_size: u32) -> bool { + if (distinct_keys_1305 >= POOL_OVERLOAD_KEYS) { return true; } + if (pool_size == 0) { return false; } + return distinct_keys_1305 >= pool_size; +} + +; --- 4. The step after a refusal --------------------------------------------------------------------- + +/// FA_DONE: answered, go on. FA_NEXT_KEY: retry on next_free's key. FA_NEXT_POOL: retry on the next +/// pool. FA_GIVE_UP: no key left; the turn ends as provider_overloaded. FA_FAIL: the request is wrong. +pub const FA_DONE : u8 = 0; +pub const FA_NEXT_KEY : u8 = 1; +pub const FA_NEXT_POOL : u8 = 2; +pub const FA_GIVE_UP : u8 = 3; +pub const FA_FAIL : u8 = 4; + +/// The bound on attempts in one turn, the failed one included: three keys of the first pool to see an +/// overload, three of the second, and two to spare. It keeps a turn from spinning through 34 keys. +pub const MAX_ATTEMPTS_PER_TURN : u32 = 8; + +/// `attempts` counts the attempts of this turn so far, this one included. `next_key` and `next_pool` +/// are next_free's answers for the current pool and for the pool list (NO_KEY when none). +pub fn step(kind: u8, attempts: u32, next_key: u32, distinct_keys_1305: u32, pool_size: u32, next_pool: u32) -> u8 { + if (kind == KE_OK) { return FA_DONE; } + if (kind >= KE_REQUEST) { return FA_FAIL; } + if (attempts >= MAX_ATTEMPTS_PER_TURN) { return FA_GIVE_UP; } + var pool_out : bool = false; + if (kind == KE_TRANSIENT) { pool_out = pool_overloaded(distinct_keys_1305, pool_size); } + if (pool_out == false) { + if (next_key != NO_KEY) { return FA_NEXT_KEY; } + } + if (next_pool != NO_KEY) { return FA_NEXT_POOL; } + return FA_GIVE_UP; +} + +/// Whether a turn that ended on this kind counts against the issue. A provider that is overloaded, +/// spent or refusing a key says nothing about the work, so it never retires the issue; a wrong request +/// does. Only KE_REQUEST charges. This narrows dispatch_exit.t27 counts_against_issue, which charges +/// every DX_PROVIDER_ERROR. +pub fn charges_issue(kind: u8) -> bool { + return kind == KE_REQUEST; +} + +; --- 5. Tests ---------------------------------------------------------------------------------------- + +test the_quota_list_is_the_closed_list_of_the_runtime { + assert is_quota_code(1113) == true; + assert is_quota_code(1308) == true; + assert is_quota_code(1311) == true; + assert is_quota_code(1312) == false; + assert is_quota_code(1313) == true; + assert is_quota_code(1321) == true; + assert is_quota_code(1322) == false; + assert is_quota_code(1307) == false; + assert is_quota_code(1112) == false; + assert is_quota_code(1114) == false; + assert is_quota_code(ZAI_RATE_LIMIT) == false; + assert is_quota_code(ZAI_OVERLOAD) == false; + assert is_quota_code(0) == false; +} + +test the_code_wins_over_the_status { + assert error_kind(429, 1305) == KE_TRANSIENT; + assert error_kind(429, 1302) == KE_TRANSIENT; + assert error_kind(429, 1113) == KE_QUOTA; + assert error_kind(429, 1316) == KE_QUOTA; + assert error_kind(200, 1308) == KE_QUOTA; + assert error_kind(400, 1305) == KE_TRANSIENT; + assert error_kind(400, 1302) == KE_TRANSIENT; + assert error_kind(429, 1312) == KE_TRANSIENT; +} + +test a_status_without_a_code_falls_in_its_kind { + assert error_kind(0, 0) == KE_TRANSIENT; + assert error_kind(200, 0) == KE_OK; + assert error_kind(399, 0) == KE_OK; + assert error_kind(400, 0) == KE_REQUEST; + assert error_kind(401, 0) == KE_AUTH; + assert error_kind(402, 0) == KE_QUOTA; + assert error_kind(403, 0) == KE_AUTH; + assert error_kind(404, 0) == KE_REQUEST; + assert error_kind(407, 0) == KE_REQUEST; + assert error_kind(408, 0) == KE_TRANSIENT; + assert error_kind(409, 0) == KE_REQUEST; + assert error_kind(428, 0) == KE_REQUEST; + assert error_kind(429, 0) == KE_TRANSIENT; + assert error_kind(430, 0) == KE_REQUEST; + assert error_kind(499, 0) == KE_REQUEST; + assert error_kind(500, 0) == KE_TRANSIENT; + assert error_kind(503, 0) == KE_TRANSIENT; + assert error_kind(529, 0) == KE_TRANSIENT; + assert error_kind(599, 0) == KE_TRANSIENT; + assert error_kind(600, 0) == KE_REQUEST; +} + +test a_transient_refusal_rests_seconds_and_a_spent_key_closes { + assert cooldown_seconds(KE_TRANSIENT, 0) == 15; + assert cooldown_seconds(KE_TRANSIENT, 1) == 30; + assert cooldown_seconds(KE_TRANSIENT, 2) == 60; + assert cooldown_seconds(KE_TRANSIENT, 3) == 120; + assert cooldown_seconds(KE_TRANSIENT, 4) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_TRANSIENT, 5) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_TRANSIENT, 4000000000) == COOL_CAP_SECONDS; + assert cooldown_seconds(KE_QUOTA, 0) == KEY_BREAK_SECONDS; + assert cooldown_seconds(KE_AUTH, 7) == KEY_BREAK_SECONDS; + assert cooldown_seconds(KE_OK, 3) == 0; + assert cooldown_seconds(KE_REQUEST, 3) == 0; + assert key_closes(KE_QUOTA) == true; + assert key_closes(KE_AUTH) == true; + assert key_closes(KE_TRANSIENT) == false; + assert key_closes(KE_OK) == false; + assert key_closes(KE_REQUEST) == false; +} + +test bits_name_keys { + assert bit(0) == 1; + assert bit(3) == 8; + assert bit(31) == 2147483648; + assert bit(32) == 0; + assert is_set(8, 3) == true; + assert is_set(8, 2) == false; + assert is_set(4294967295, 31) == true; + assert is_set(4294967295, 32) == false; +} + +test the_next_key_is_never_the_one_that_failed { + // the #9967 case: pool 2 has 10 keys, key 9 answered 1305; the next try wraps to key 0 + assert next_free(bit(9), 0, 10, 9) == 0; + // even when the failed key is not marked tried, it is skipped + assert next_free(0, 0, 10, 9) == 0; + assert next_free(0, 0, 10, 4) == 5; + // a pool of one key has no other key + assert next_free(0, 0, 1, 0) == NO_KEY; + // tried and resting keys are both skipped + assert next_free(bit(5), bit(6), 10, 4) == 7; + assert next_free(bit(0) | bit(1), 0, 3, 2) == NO_KEY; + // a fresh turn starts at key 0 + assert next_free(0, 0, 24, NO_KEY) == 0; + assert next_free(bit(0), 0, 24, NO_KEY) == 1; + // every key but the failed one is taken + assert next_free(4294967295 - bit(9), 0, 10, 9) == NO_KEY; + // an empty pool has nothing + assert next_free(0, 0, 0, NO_KEY) == NO_KEY; + // `last` just past the end is out of range: the scan starts at 0 + assert next_free(0, 0, 10, 10) == 0; + // a full mask of POOL_MAX keys wraps from the top index + assert next_free(0, 0, 32, 31) == 0; + assert next_free(0, 0, 32, NO_KEY) == 0; + assert next_free(0, 4294967295 - bit(31), 32, 30) == 31; + // a list longer than POOL_MAX does not fit a mask and is refused + assert next_free(0, 0, 33, NO_KEY) == NO_KEY; + assert next_free(0, 0, 40, 31) == NO_KEY; +} + +test an_overload_seen_on_three_keys_is_the_pools_model { + assert pool_overloaded(0, 24) == false; + assert pool_overloaded(2, 24) == false; + assert pool_overloaded(3, 24) == true; + assert pool_overloaded(16, 24) == true; + assert pool_overloaded(2, 2) == true; + assert pool_overloaded(1, 2) == false; + assert pool_overloaded(1, 1) == true; + assert pool_overloaded(0, 0) == false; +} + +/// The conformance table: one assert per row. Columns: kind, attempts, next key, distinct keys that +/// answered 1305 in the window, pool size, next pool. +test conformance_rows { + // r01 answered: go on + assert step(KE_OK, 1, 5, 0, 24, 1) == FA_DONE; + // r02 a wrong request fails, whatever keys are left + assert step(KE_REQUEST, 1, 5, 0, 24, 1) == FA_FAIL; + // r03 an unknown kind is read as a wrong request + assert step(9, 1, 5, 0, 24, 1) == FA_FAIL; + // r04 the #9967 case: first 1305 on pool 2, keys left: next key, not the same key + assert step(KE_TRANSIENT, 1, 0, 1, 10, 0) == FA_NEXT_KEY; + // r05 a rate limit (1302) on a fresh pool: next key + assert step(KE_TRANSIENT, 1, 3, 0, 24, 1) == FA_NEXT_KEY; + // r06 the third distinct key said 1305: the model is overloaded, next pool + assert step(KE_TRANSIENT, 3, 4, 3, 24, 1) == FA_NEXT_POOL; + // r07 the 14:08 burst: overloaded and no other pool: give up, do not burn the rest + assert step(KE_TRANSIENT, 3, 4, 3, 24, NO_KEY) == FA_GIVE_UP; + // r08 a spent quota on one key does not mean an overloaded pool: next key + assert step(KE_QUOTA, 1, 2, 5, 24, 1) == FA_NEXT_KEY; + // r09 a refused key: next key + assert step(KE_AUTH, 1, 2, 0, 24, 1) == FA_NEXT_KEY; + // r10 no key left in the pool: next pool + assert step(KE_TRANSIENT, 2, NO_KEY, 1, 24, 1) == FA_NEXT_POOL; + // r11 no key and no pool left: give up + assert step(KE_QUOTA, 2, NO_KEY, 0, 24, NO_KEY) == FA_GIVE_UP; + // r12 the attempt bound stops a turn with keys still left + assert step(KE_TRANSIENT, MAX_ATTEMPTS_PER_TURN, 4, 0, 24, 1) == FA_GIVE_UP; + // r13 one attempt below the bound still moves on + assert step(KE_TRANSIENT, MAX_ATTEMPTS_PER_TURN - 1, 4, 0, 24, 1) == FA_NEXT_KEY; + // r14 a two-key pool where both said 1305 is overloaded + assert step(KE_TRANSIENT, 2, NO_KEY, 2, 2, 0) == FA_NEXT_POOL; + // r15 a 503 with no 1305 seen keeps to the pool + assert step(KE_TRANSIENT, 2, 7, 0, 24, 1) == FA_NEXT_KEY; + // r16 the bound beats a wrong request? no: a wrong request fails first + assert step(KE_REQUEST, MAX_ATTEMPTS_PER_TURN, 4, 0, 24, 1) == FA_FAIL; + // r17 an answer after many attempts is still an answer + assert step(KE_OK, MAX_ATTEMPTS_PER_TURN, NO_KEY, 9, 24, NO_KEY) == FA_DONE; +} + +test an_overloaded_provider_never_retires_the_issue { + assert charges_issue(KE_TRANSIENT) == false; + assert charges_issue(KE_QUOTA) == false; + assert charges_issue(KE_AUTH) == false; + assert charges_issue(KE_REQUEST) == true; + assert charges_issue(KE_OK) == false; + assert charges_issue(9) == false; +} + +; --- 6. Invariants ----------------------------------------------------------------------------------- + +invariant the_kinds_and_steps_are_distinct { + assert KE_OK == 0 and KE_TRANSIENT == 1 and KE_QUOTA == 2 and KE_AUTH == 3 and KE_REQUEST == 4; + assert FA_DONE == 0 and FA_NEXT_KEY == 1 and FA_NEXT_POOL == 2 and FA_GIVE_UP == 3 and FA_FAIL == 4; +} + +/// A transient rest is always shorter than the long break, so a 1305 can never close a key. +invariant a_rest_is_shorter_than_a_break { + assert COOL_BASE_SECONDS > 0 and COOL_CAP_SECONDS == COOL_BASE_SECONDS * 16; + assert COOL_CAP_SECONDS < KEY_BREAK_SECONDS and KEY_BREAK_SECONDS == 1800; +} + +/// The bound leaves room to see an overload in one pool and still try the next: the overload is +/// detected before the bound stops the turn. +invariant the_bound_leaves_room_for_a_second_pool { + assert MAX_ATTEMPTS_PER_TURN > POOL_OVERLOAD_KEYS * 2; + assert POOL_OVERLOAD_KEYS >= 2 and OVERLOAD_WINDOW_SECONDS > 0; +} + +/// NO_KEY is no index a mask can hold. +invariant no_key_is_out_of_range { + assert POOL_MAX == 32 and NO_KEY >= POOL_MAX; + assert ZAI_RATE_LIMIT == 1302 and ZAI_OVERLOAD == 1305; +} diff --git a/trios/agent-server/specs/queen/key_vault.t27 b/trios/agent-server/specs/queen/key_vault.t27 new file mode 100644 index 0000000000..341c272cb4 --- /dev/null +++ b/trios/agent-server/specs/queen/key_vault.t27 @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: Apache-2.0 +; specs/queen/key_vault.t27 -- every provider key lives in the Queen's one store; every consumer leases from it +; Owner, 2026-10-18 (translated): all API keys -- the Queen's pools and every contributor's key -- live +; safely in ONE place and are redistributed among everyone who uses them. The Mac reviewer must not run +; on its own keys outside the shared pool. +; Measured 2026-10-19 (names only, no value read): +; - the Queen's pools are Railway variables: pool 1 (TRIOS_QUEEN_WORKER_API_KEY and _1.._24) and pool 2 +; (TRIOS_QUEEN_WORKER_POOL_2_API_KEY and _1.._9), copied to trios-agent-server, trios-bee-runner-1 +; and trios-bee-runner-2 -- three copies of each key, all inside the Queen's own services; +; - contributor keys are rows in the Queen's Postgres, sealed with AES-256-GCM (trios +; queen-contributor-keys.ts); +; - the Mac reviewer (tools/bees/reviewer.py, launchd ai.t27.reviewer-bees) reads five more z.ai keys, +; ZAI_KEY_1, _2, _3, _5, _6, from ~/.claude/.env: a second home the Queen cannot see. Its reviews and +; the Queen's bees can hit one key at once, and neither side knows the other holds it; +; - at the same time the Queen ran 70 bees with 354 unreviewed dispatches and about 5 reviews an hour: +; review, not lanes, is the bottleneck. +; The rule here: +; 1. a key the Queen holds (a service variable or a sealed contributor row) is in the store; a key one +; consumer holds alone is private, and nothing serves from it (may_serve); +; 2. a consumer leases from the store. One that holds private keys while a lease is configured is +; refused: two homes for one key. One with no lease yet runs on its own keys and says so, only +; until the lease exists (consumer_start); +; 3. the lanes are shared by demand. A z.ai key carries ZAI_REQUESTS_PER_KEY requests +; (reviewer_sizing.t27). Reviews that hold or wait for a lane get the lanes first, never fewer than +; one key's worth; bees get the rest (review_share, worker_share). A free lane goes to a waiting +; review before a bee. Inside review, no holder takes more than its share of the holders, except +; lanes nobody else is asking for, which are lent (holder_cap, lease_answer); +; 4. a lease covers one request. The consumer never sees a key's value: the store makes the request on +; a lane and fails over to the next lane on a refusal (lease_step, after key_failover.t27 +; error_kind). A held lease is never taken back; the share moves when leases end. +; The admission of new bees while reviews wait is a sibling card, not this one. Not here: the wiring +; (trios: an authenticated lease route on the Queen; the Mac: one switch). ASCII only (L3). +; phi^2 + 1/phi^2 = 3 | TRINITY + +module QueenKeyVault; + +use queen::key_failover::error_kind; +use queen::key_failover::KE_OK; +use queen::key_failover::KE_TRANSIENT; +use queen::key_failover::KE_QUOTA; +use queen::key_failover::KE_AUTH; +use queen::key_failover::KE_REQUEST; +use queen::key_failover::COOL_BASE_SECONDS; +use queen::reviewer_sizing::ZAI_REQUESTS_PER_KEY; +use queen::reviewer_sizing::U32_MAX; + +; --- 1. Where a key lives ---------------------------------------------------------------------------- + +/// SRC_STORE: a key the Queen holds -- a variable of her own services or a sealed contributor row. +/// SRC_PRIVATE: a key one consumer holds alone, outside the store (a dotenv file on a Mac, another +/// repository's Actions secret). Any other source is unknown. +pub const SRC_STORE : u8 = 0; +pub const SRC_PRIVATE : u8 = 1; + +/// Only the store serves a lease. A private key is never leased, never pooled, never read as the +/// store's: the Queen cannot see who else holds it, so she cannot share it. +pub fn may_serve(src: u8) -> bool { + return src == SRC_STORE; +} + +; --- 2. Who may start -------------------------------------------------------------------------------- + +/// START_LEASE: the consumer leases from the store. START_OWN_KEYS: no lease is configured yet; it runs +/// on its own keys and says so on every start, until the lease exists. START_REFUSE: it may not start. +pub const START_LEASE : u8 = 0; +pub const START_OWN_KEYS : u8 = 1; +pub const START_REFUSE : u8 = 2; + +/// `private_keys` is the count of provider keys the consumer holds itself, `lease_on` whether a lease +/// from the store is configured. Both at once is two homes for one key: refused, so moving the keys +/// into the store is the only way on. Neither is a consumer with no key at all. +pub fn consumer_start(private_keys: u32, lease_on: bool) -> u8 { + if (lease_on) { + if (private_keys > 0) { return START_REFUSE; } + return START_LEASE; + } + if (private_keys > 0) { return START_OWN_KEYS; } + return START_REFUSE; +} + +/// Whether the start is logged as running outside the store. +pub fn start_warns(answer: u8) -> bool { + return answer == START_OWN_KEYS; +} + +; --- 3. How the lanes are shared --------------------------------------------------------------------- + +/// LEASE_REVIEW: a review's model call (the round reviewer, the Queen app's review, the Mac reviewer). +/// LEASE_WORKER: a bee's turn. +pub const LEASE_REVIEW : u8 = 0; +pub const LEASE_WORKER : u8 = 1; + +/// Reviews never get fewer lanes than one key carries, even while none is asking: the next review +/// finds a lane without waiting for a bee to end. Written as a literal and pinned to +/// ZAI_REQUESTS_PER_KEY by an invariant: gen-c emits a const set from an imported const before the +/// import, and that C does not compile. +pub const REVIEW_FLOOR_LANES : u32 = 2; + +/// The review holders that ask the store for lanes: the round reviewer, the Queen app's review (with +/// the shaper, through the free lane), and the Mac reviewer through the lease route. Chosen, not +/// measured: one share each. +pub const REVIEW_HOLDERS : u32 = 3; + +/// The lanes `keys` z.ai keys carry. Saturates rather than wraps. +pub fn lanes_of(keys: u32) -> u32 { + if (keys > U32_MAX / ZAI_REQUESTS_PER_KEY) { return U32_MAX; } + return keys * ZAI_REQUESTS_PER_KEY; +} + +/// The lanes reviews may hold: what they ask for -- the reviews holding a lane and the ones waiting +/// for one (`review_demand`) -- never below REVIEW_FLOOR_LANES, never above all the lanes. Demand is +/// lanes asked for now, not the backlog of unreviewed work: a backlog of 354 rows that 8 reviewers +/// serve wants 8 lanes, and giving it 68 would leave 60 idle while bees wait. +/// (Mutation: `<` to `<=` and `>` to `>=` below are equivalent -- at equality both branches give the +/// same number.) +pub fn review_share(total: u32, review_demand: u32) -> u32 { + var want : u32 = review_demand; + if (want < REVIEW_FLOOR_LANES) { want = REVIEW_FLOOR_LANES; } + if (want > total) { return total; } + return want; +} + +/// The lanes bees may hold: all the rest. +pub fn worker_share(total: u32, review_demand: u32) -> u32 { + return total - review_share(total, review_demand); +} + +/// One review holder's part of the review share: the share split over the holders, rounded up, so a +/// share of one lane still lets one holder in. 0 when the share is 0 (0 / holders, no guard needed). +/// No holders is one holder: the whole share. +pub fn holder_cap(share: u32, holders: u32) -> u32 { + if (holders == 0) { return share; } + var part : u32 = share / holders; + if (share % holders != 0) { part = part + 1; } + return part; +} + +/// LA_GRANT: take a lane now. LA_WAIT: none for this asker now; ask again after retry_after_seconds. +/// LA_REFUSE: never, whatever the lanes. +pub const LA_GRANT : u8 = 0; +pub const LA_WAIT : u8 = 1; +pub const LA_REFUSE : u8 = 2; + +/// One ask for a lease. `held` is the lanes this asker already holds: one review holder's, or all the +/// bees' together for LEASE_WORKER. `free_lanes` is the lanes no request holds now, `total` all the +/// lanes. `review_holding` and `review_waiting` are the reviews holding and waiting for a lane; a +/// review's own ask counts as waiting. (Mutation: `>` to `>=` in the saturating sum is equivalent -- +/// at equality the sum is U32_MAX either way.) +/// - an unknown kind, or a source other than the store, is refused; +/// - a review takes a lane while its holder is under holder_cap. Over it, it still takes a lane when +/// more than holder_cap lanes are free: nobody else is asking for those, so they are lent, and +/// holder_cap lanes stay free for the other holders; +/// - a bee waits while every free lane is a waiting review's, and while the bees hold their share. +pub fn lease_answer(kind: u8, src: u8, held: u32, free_lanes: u32, total: u32, review_holding: u32, review_waiting: u32) -> u8 { + if (kind > LEASE_WORKER) { return LA_REFUSE; } + if (may_serve(src) == false) { return LA_REFUSE; } + if (free_lanes == 0) { return LA_WAIT; } + var demand : u32 = review_holding; + if (demand > U32_MAX - review_waiting) { demand = U32_MAX; } else { demand = demand + review_waiting; } + if (kind == LEASE_REVIEW) { + var cap : u32 = holder_cap(review_share(total, demand), REVIEW_HOLDERS); + if (held < cap) { return LA_GRANT; } + if (free_lanes > cap) { return LA_GRANT; } + return LA_WAIT; + } + if (free_lanes <= review_waiting) { return LA_WAIT; } + if (held >= worker_share(total, demand)) { return LA_WAIT; } + return LA_GRANT; +} + +/// Seconds a waiting asker waits before it asks again: the short rest key_failover.t27 gives a +/// transient refusal, so a wait never outlasts a lane's own rest. 0 for any other answer. +pub fn retry_after_seconds(answer: u8) -> u32 { + if (answer == LA_WAIT) { return COOL_BASE_SECONDS; } + return 0; +} + +; --- 4. One leased request --------------------------------------------------------------------------- + +/// The store makes the request on a lane; the consumer gets the answer, never the key. A lease covers +/// one request and lasts at most LEASE_SECONDS: the Mac reviewer's own request timeout (API_TIMEOUT_MS +/// 600000 in tools/bees/reviewer.py), so the store never cuts a request the consumer still waits for. +pub const LEASE_SECONDS : u32 = 600; + +/// A refused request tries at most LEASE_TRIES lanes, each once: the free lane's MAX_LANES in trios +/// queen-free-lane.ts. +pub const LEASE_TRIES : u32 = 3; + +/// LS_ANSWER: hand this answer to the consumer. LS_NEXT_LANE: try the request on the next lane. +pub const LS_ANSWER : u8 = 0; +pub const LS_NEXT_LANE : u8 = 1; + +/// `status` and `zai_code` are the lane's answer (key_failover.t27 error_kind), `tries` the lanes tried +/// so far, this one included. A refusal of the lane (transient, quota, auth) moves to the next lane +/// while tries are left; an answer or a wrong request goes back as it is, because every lane would +/// answer a wrong request the same. +pub fn lease_step(status: u32, zai_code: u32, tries: u32) -> u8 { + var kind : u8 = error_kind(status, zai_code); + if (kind == KE_OK) { return LS_ANSWER; } + if (kind == KE_REQUEST) { return LS_ANSWER; } + if (tries >= LEASE_TRIES) { return LS_ANSWER; } + return LS_NEXT_LANE; +} + +; --- 5. Tests ----------------------------------------------------------------------------------------- + +test only_the_store_serves { + assert may_serve(SRC_STORE) == true; + // negative controls: a private key, an unknown source + assert may_serve(SRC_PRIVATE) == false; + assert may_serve(7) == false; +} + +test a_consumer_with_two_homes_is_refused { + // the Mac reviewer after the move: the lease, no key of its own + assert consumer_start(0, true) == START_LEASE; + // the Mac reviewer today: five keys in ~/.claude/.env, no lease yet + assert consumer_start(5, false) == START_OWN_KEYS; + assert start_warns(consumer_start(5, false)) == true; + // negative control: the lease configured while the five keys stay in the dotenv + assert consumer_start(5, true) == START_REFUSE; + assert consumer_start(1, true) == START_REFUSE; + // no key and no lease: nothing to run on + assert consumer_start(0, false) == START_REFUSE; + assert start_warns(START_LEASE) == false; + assert start_warns(START_REFUSE) == false; +} + +test lanes_follow_the_keys { + // pools 1 and 2: 24 + 10 keys + assert lanes_of(34) == 68; + assert lanes_of(0) == 0; + assert lanes_of(1) == REVIEW_FLOOR_LANES; + assert lanes_of(2147483647) == 4294967294; + assert lanes_of(2147483648) == U32_MAX; +} + +test reviews_get_the_lanes_they_ask_for_first { + // idle: one key's worth stays with review, bees get the rest + assert review_share(68, 0) == 2; + assert worker_share(68, 0) == 66; + // 8 reviewers busy, 4 waiting: 12 lanes, not the 354-row backlog + assert review_share(68, 12) == 12; + assert worker_share(68, 12) == 56; + // more demand than lanes: review takes them all, bees none + assert review_share(68, 354) == 68; + assert worker_share(68, 354) == 0; + // fewer lanes than the floor + assert review_share(1, 0) == 1; + assert worker_share(1, 0) == 0; + assert review_share(0, 5) == 0; + assert worker_share(0, 5) == 0; +} + +test a_holder_takes_its_part_of_the_share { + assert holder_cap(12, 3) == 4; + assert holder_cap(13, 3) == 5; + assert holder_cap(2, 3) == 1; + assert holder_cap(1, 3) == 1; + assert holder_cap(0, 3) == 0; + assert holder_cap(68, 1) == 68; + assert holder_cap(68, 0) == 68; + assert holder_cap(U32_MAX, 2) == 2147483648; +} + +/// The conformance table: one assert per row. Columns: kind, source, held, free lanes, total lanes, +/// reviews holding, reviews waiting. +test conformance_rows { + // r01 the Mac's first review, the pool idle: granted + assert lease_answer(LEASE_REVIEW, SRC_STORE, 0, 60, 68, 0, 1) == LA_GRANT; + // r02 negative control: a private source is refused, lanes or not + assert lease_answer(LEASE_REVIEW, SRC_PRIVATE, 0, 60, 68, 0, 1) == LA_REFUSE; + // r03 negative control: an unknown kind is refused + assert lease_answer(2, SRC_STORE, 0, 60, 68, 0, 1) == LA_REFUSE; + assert lease_answer(9, SRC_STORE, 0, 60, 68, 0, 1) == LA_REFUSE; + // r04 no lane free: wait, review or bee + assert lease_answer(LEASE_REVIEW, SRC_STORE, 0, 0, 68, 10, 1) == LA_WAIT; + assert lease_answer(LEASE_WORKER, SRC_STORE, 0, 0, 68, 0, 0) == LA_WAIT; + // r05 a holder at its cap (share 12, cap 4) with 3 lanes free: the rest are the others', wait + assert lease_answer(LEASE_REVIEW, SRC_STORE, 4, 3, 68, 11, 1) == LA_WAIT; + // r06 the same holder with 10 lanes free: nobody else asks for them, lent + assert lease_answer(LEASE_REVIEW, SRC_STORE, 4, 10, 68, 11, 1) == LA_GRANT; + // r07 the cap boundary: free lanes equal to the cap are kept for the others + assert lease_answer(LEASE_REVIEW, SRC_STORE, 4, 4, 68, 11, 1) == LA_WAIT; + assert lease_answer(LEASE_REVIEW, SRC_STORE, 4, 5, 68, 11, 1) == LA_GRANT; + // r08 a holder under its cap takes the last free lane + assert lease_answer(LEASE_REVIEW, SRC_STORE, 3, 1, 68, 11, 1) == LA_GRANT; + // r09 negative control: a bee asks for the one free lane while a review waits: the review's + assert lease_answer(LEASE_WORKER, SRC_STORE, 50, 1, 68, 8, 1) == LA_WAIT; + assert lease_answer(LEASE_WORKER, SRC_STORE, 50, 3, 68, 8, 3) == LA_WAIT; + // r10 more free lanes than waiting reviews: a bee under its share gets one + assert lease_answer(LEASE_WORKER, SRC_STORE, 50, 4, 68, 8, 3) == LA_GRANT; + // r11 the bees at their share (68 - 11 = 57): wait + assert lease_answer(LEASE_WORKER, SRC_STORE, 57, 4, 68, 8, 3) == LA_WAIT; + assert lease_answer(LEASE_WORKER, SRC_STORE, 56, 4, 68, 8, 3) == LA_GRANT; + // r12 review idle: bees may fill all but one key's lanes + assert lease_answer(LEASE_WORKER, SRC_STORE, 65, 3, 68, 0, 0) == LA_GRANT; + assert lease_answer(LEASE_WORKER, SRC_STORE, 66, 2, 68, 0, 0) == LA_WAIT; + // r13 demand past every lane: no bee lease at all + assert lease_answer(LEASE_WORKER, SRC_STORE, 0, 10, 68, 60, 300) == LA_WAIT; + // r14 a review with a saturating count still gets an answer, not a wrap + assert lease_answer(LEASE_REVIEW, SRC_STORE, 0, 5, 68, U32_MAX, U32_MAX) == LA_GRANT; +} + +test a_wait_says_when_to_ask_again { + assert retry_after_seconds(LA_WAIT) == COOL_BASE_SECONDS; + assert retry_after_seconds(LA_GRANT) == 0; + assert retry_after_seconds(LA_REFUSE) == 0; +} + +test a_refused_lane_hands_the_request_to_the_next { + // answered + assert lease_step(200, 0, 1) == LS_ANSWER; + // 1305 overload, 1302 rate limit, a spent quota, a refused key: next lane + assert lease_step(429, 1305, 1) == LS_NEXT_LANE; + assert lease_step(429, 1302, 2) == LS_NEXT_LANE; + assert lease_step(429, 1113, 1) == LS_NEXT_LANE; + assert lease_step(401, 0, 1) == LS_NEXT_LANE; + assert lease_step(0, 0, 1) == LS_NEXT_LANE; + assert lease_step(503, 0, 2) == LS_NEXT_LANE; + // the bound: the third lane's refusal goes back to the consumer + assert lease_step(429, 1305, LEASE_TRIES) == LS_ANSWER; + assert lease_step(429, 1305, LEASE_TRIES + 1) == LS_ANSWER; + // negative control: a wrong request is the consumer's, never retried on another lane + assert lease_step(400, 0, 1) == LS_ANSWER; + assert lease_step(404, 0, 1) == LS_ANSWER; +} + +; --- 6. Invariants ----------------------------------------------------------------------------------- + +invariant the_codes_are_distinct { + assert SRC_STORE == 0 and SRC_PRIVATE == 1; + assert START_LEASE == 0 and START_OWN_KEYS == 1 and START_REFUSE == 2; + assert LEASE_REVIEW == 0 and LEASE_WORKER == 1; + assert LA_GRANT == 0 and LA_WAIT == 1 and LA_REFUSE == 2; + assert LS_ANSWER == 0 and LS_NEXT_LANE == 1; +} + +/// The review floor is one key's lanes, and a holder's part of it is never 0. +invariant the_floor_is_one_key { + assert REVIEW_FLOOR_LANES == ZAI_REQUESTS_PER_KEY and REVIEW_FLOOR_LANES >= 1; + assert REVIEW_HOLDERS == 3 and holder_cap(REVIEW_FLOOR_LANES, REVIEW_HOLDERS) >= 1; +} + +/// A lease outlasts a review's model call and a wait is shorter than a lease; the tries leave room for +/// a second and a third lane. +invariant the_bounds_fit { + assert LEASE_SECONDS == 600 and COOL_BASE_SECONDS < LEASE_SECONDS; + assert LEASE_TRIES == 3 and LEASE_TRIES > 1; + assert KE_TRANSIENT != KE_OK and KE_QUOTA != KE_REQUEST and KE_AUTH != KE_REQUEST; +} diff --git a/trios/agent-server/specs/queen/key_vault.wasm b/trios/agent-server/specs/queen/key_vault.wasm new file mode 100644 index 0000000000..64aba70a34 Binary files /dev/null and b/trios/agent-server/specs/queen/key_vault.wasm differ diff --git a/trios/tools/route-guard-audit.mjs b/trios/tools/route-guard-audit.mjs index 7ecc1823aa..84533e5396 100644 --- a/trios/tools/route-guard-audit.mjs +++ b/trios/tools/route-guard-audit.mjs @@ -120,6 +120,11 @@ export const DEFAULT_ALLOWLIST = [ reason: 'own capability - a server-to-server route for the app render proxy, with no browser Origin; every request is refused unless its bearer equals QUEEN_CONTRIBUTOR_PROXY_TOKEN (at least 32 bytes, compared with timingSafeEqual) and it carries a verified x-queen-contributor-id, and the route is off while that token is unset (src/api/routes/queen-contributor-keys.ts, tests/api/queen-contributor-keys.test.ts)', }, + { + path: '/queen/zai-lease', + reason: + 'own capability - the Mac reviewer\'s z.ai lease (specs/queen/key_vault.t27), called by a bee process with no browser Origin; every request is refused unless its bearer equals TRIOS_QUEEN_REVIEW_LEASE_TOKEN (at least 32 bytes, sha256 digests compared with timingSafeEqual), the route answers 503 while that token is unset, and it forwards a model request on a Queen key without ever returning a key (src/api/routes/queen-lease.ts)', + }, { path: '/vault', reason: