From 9b2f9069c2ea66c4eef39a24bb826f522d3c4389 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 17 Sep 2026 18:44:27 +0700 Subject: [PATCH] fix(queen): every bound on keys and pools said nothing when it bound, so stop counting to a number The key list was read by counting: the unsuffixed variable, then _2 up to a constant. The constant was 16, then 1024, and both were the same mistake - the next variable past it looks configured in an editor and is read by nothing. Pools were counted the same way, to eight. The operator now wants to connect credentials without an upper edge, and any new constant would only move the place where a variable silently stops being read. So the names are READ FROM THE ENVIRONMENT. keysFor() asks which suffixed variables exist and takes them in numeric order (17 before 1024 before 4096), the unsuffixed name first, so every index already written still names the same key. A suffix must be a plain integer of two or more; _1, _02 and _x are not second names for an existing slot and are ignored rather than guessed at. Numbered pools are discovered the same way. The cost is the size of the environment instead of the size of a range. What is left is arithmetic, not policy: - a pool is cut at MAX_KEYS_PER_POOL = POOL_KEY_STRIDE - 1 (9999), because the next pool's durable key_index begins at the stride and that stride is already written in production rows, so it does not move; - a pool number above MAX_POOL_NUMBER cannot be addressed by a 32-bit key_index and is reported by endpointPoolProblems() instead of being dropped silently. The worker ceiling moves from 1024 to WORKER_SANITY_BOUND = 1_000_000 in the three places that must agree (queen-dispatch.ts and the two byte-identical QueenDelegation.swift copies). This reverses a choice made this morning, so the reasoning is stated plainly: as a typo guard 1024 protected little. An operator who meant 50 and typed 5000 over two thousand connected lanes got 1024 bees instead of 2000, and either ends a container sized for fifty. A mistyped value is bounded by the credential list, because dispatch refuses when every lane is taken. The constant now only stops a value that is not a number of bees at all. The default of four, and TRIOS_QUEEN_MAX_WORKERS as the operator's control, are untouched. Not claimed: that one container can run thousands of bees. Memory, worktree disk, the GitHub API quota, the review backlog and the supply of eligible issues bind long before any of these numbers. This change only guarantees that none of them is a constant in this file. Dispatch tests: 87 pass (82 before). Typecheck clean. The two Swift copies are byte-identical (cmp); I cannot build queend here, so queen-core-sync and queen-core need to be run where Swift for Linux is available. Co-Authored-By: Claude Fable 5.1 --- .../server/src/api/services/queen-dispatch.ts | 116 +++++++++++++----- .../server/tests/api/queen-dispatch.test.ts | 116 +++++++++++++++--- .../Sources/QueenPolicy/QueenDelegation.swift | 13 +- trios/rings/SR-00/QueenDelegation.swift | 13 +- 4 files changed, 203 insertions(+), 55 deletions(-) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 3423171b33..348e4c6693 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -232,23 +232,26 @@ export interface WorkerProvider { * its padding, and handing the trimmed form out keeps the count and the * selection - which both read this list - from ever disagreeing. * - * The suffix runs to MAX_KEYS_PER_POOL. It used to stop at 16, and 16 was never - * a measurement: it was where the loop happened to end, equal by coincidence to - * the policy ceiling on bees. They are different quantities. The ceiling bounds - * how many bees RUN; this bounds how many credentials the rotation may SPREAD - * them over, and a free key refuses its third concurrent request in under half - * a second (`1302`), so a swarm at the ceiling wants more keys than bees, not - * the same number. A seventeenth variable used to be read by nothing and - * reported by nothing, which is the zero-length-key trap again: it looks - * configured and supplies nothing. Widening the list does not widen the swarm - - * `queenWorkerLimit` still decides that. - * - * 1024 rather than a round thousand so that this bound can never sit BELOW a - * worker ceiling expressed as a power of two: with one lane per credential a - * swarm of N bees needs N keys, and a key bound under the worker bound would be - * a second, hidden ceiling that reports nothing when it is hit. + * The suffixes are READ FROM THE ENVIRONMENT, not counted up to a number. The + * loop used to stop at 16 and then at 1024, and both were the same kind of + * mistake: a bound that says nothing when it binds. `_17` was read by nothing + * while the loop ended at 16, and `_1025` would have been read by nothing + * after it. Asking the environment which suffixed names exist has no such + * edge - every variable an operator can see in the editor is a variable this + * function reads - and it costs the size of the environment rather than the + * size of a range. + * + * Order is the numeric suffix, the unsuffixed name first, so every index a + * deployment has already written still names the same key. A suffix must be a + * plain integer of two or more (`_2`, `_17`, `_4096`): `_02`, `_1` and `_x` + * are not key names and are ignored rather than guessed at. + * + * What remains is a consequence, not a policy: a pool is cut at + * MAX_KEYS_PER_POOL because the durable index of the next pool starts at + * POOL_KEY_STRIDE, and that stride is already written in production rows. + * Widening the list does not widen the swarm - `queenWorkerLimit` decides that. */ -export const MAX_KEYS_PER_POOL = 1024 +export const MAX_KEYS_PER_POOL = 9_999 function keysFor(envVar: string): string[] { const keys: string[] = [] @@ -260,8 +263,16 @@ function keysFor(envVar: string): string[] { keys.push(trimmed) } admit(process.env[envVar]) - for (let i = 2; i <= MAX_KEYS_PER_POOL; i++) { - admit(process.env[`${envVar}_${i}`]) + const prefix = `${envVar}_` + const suffixes = Object.keys(process.env) + .filter((name) => name.startsWith(prefix)) + .map((name) => name.slice(prefix.length)) + .filter((suffix) => /^[1-9]\d*$/.test(suffix) && Number(suffix) >= 2) + .map(Number) + .sort((a, b) => a - b) + for (const suffix of suffixes) { + if (keys.length >= MAX_KEYS_PER_POOL) break + admit(process.env[`${envVar}_${suffix}`]) } return keys } @@ -298,18 +309,25 @@ function workerLanesFor(provider: string): number { * dispatch, and promising less hides capacity that was paid for. * * So both sides now read the SAME environment variable, with the same default - * and the same ceiling. That ceiling is 1024 and it is a guard against a typo, - * not a working value: capacity is decided by connected credentials times - * lanes and by the number the operator sets. It used to be 16, which sat below - * the credentials a deployment could connect once endpoint pools existed, and - * said nothing when it bound. Review cost is still linear in running workers - - * that is why the operator picks the VALUE by measurement. The Swift side - * records the full reasoning. + * and the same ceiling. That ceiling is WORKER_SANITY_BOUND and it no longer + * bounds a real deployment: capacity is decided by connected credentials times + * lanes and by the number the operator sets. It was 16, then 1024, and both + * times it ended up sitting below credentials a deployment could connect and + * saying nothing when it bound. As a guard against a typo 1024 protected + * little: an operator who meant 50 and typed 5000 over two thousand connected + * lanes got 1024 bees instead of 2000, and either number ends a container + * sized for fifty. What actually bounds a mistyped value is the credential + * list - dispatch refuses when every lane is taken - so the constant only has + * to stop a value that is not a number of bees at all. Review cost is still + * linear in running workers, which is why the operator picks the VALUE by + * measurement. The Swift side records the full reasoning. */ -function queenWorkerLimit(): number { +export const WORKER_SANITY_BOUND = 1_000_000 + +export function queenWorkerLimit(): number { const parsed = Number(process.env.TRIOS_QUEEN_MAX_WORKERS) if (!Number.isInteger(parsed) || parsed < 1) return 4 - return Math.min(parsed, 1024) + return Math.min(parsed, WORKER_SANITY_BOUND) } /** @@ -502,13 +520,13 @@ function availableKeyIndex( * sixteen bees while three working NVIDIA keys sat unused next to them. * * So the FIRST pool is exactly the variables that already exist, unchanged in - * name and meaning, and further pools are numbered from 2: + * name and meaning, and further pools are numbered from 2, as many as exist: * * TRIOS_QUEEN_WORKER_POOL__BASE_URL the endpoint (required) * TRIOS_QUEEN_WORKER_POOL__MODEL the model THAT endpoint serves (required) * TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai * TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window - * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... (MAX_KEYS_PER_POOL) + * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2, _API_KEY_3 ... (any number of them) * * A pool is REMOTE credentials by definition. A local Ollama stays what it was * measured to be - one inference slot - and does not join a credential pool, @@ -521,7 +539,6 @@ function availableKeyIndex( * breakdown (#1308) an exact statement: credentials x lanes, bounded by the * policy ceiling. */ -const MAX_ENDPOINT_POOLS = 8 const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) /** @@ -539,6 +556,35 @@ const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) */ export const POOL_KEY_STRIDE = 10_000 +/** + * The numbers of the pools someone has started to configure, ascending. + * + * Discovered from the environment for the same reason the key suffixes are: + * "up to eight" was a bound nobody chose, and a ninth pool would have been + * configured in the editor and read by nothing. The only ceiling left is + * arithmetic - `key_index` is a 32-bit column and pool n starts at + * (n - 1) * POOL_KEY_STRIDE - so a pool number past MAX_POOL_NUMBER cannot be + * named durably and is reported by `endpointPoolProblems` instead of being + * silently dropped. + */ +export const MAX_POOL_NUMBER = Math.floor(2_147_483_647 / POOL_KEY_STRIDE) - 1 + +function configuredPoolNumbers(): { usable: number[]; tooLarge: number[] } { + const numbers = new Set() + for (const name of Object.keys(process.env)) { + const match = + /^TRIOS_QUEEN_WORKER_POOL_([1-9]\d*)_(BASE_URL|API_KEY(_[1-9]\d*)?)$/.exec( + name, + ) + if (match && process.env[name]?.trim()) numbers.add(Number(match[1])) + } + const sorted = [...numbers].filter((n) => n >= 2).sort((a, b) => a - b) + return { + usable: sorted.filter((n) => n <= MAX_POOL_NUMBER), + tooLarge: sorted.filter((n) => n > MAX_POOL_NUMBER), + } +} + interface EndpointPool { /** 1-based; 1 is the pool made of the unnumbered variables. */ number: number @@ -575,7 +621,13 @@ function contextWindowFrom(raw: string | undefined): number { */ export function endpointPoolProblems(): string[] { const problems: string[] = [] - for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const discovered = configuredPoolNumbers() + for (const n of discovered.tooLarge) { + problems.push( + `${poolVariable(n, 'BASE_URL')} names pool ${n}, above the ${MAX_POOL_NUMBER} a durable key index can address`, + ) + } + for (const n of discovered.usable) { const baseUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) const keys = keysFor(poolVariable(n, 'API_KEY')) if (!baseUrl && keys.length === 0) continue @@ -632,7 +684,7 @@ function configuredEndpointPools(): EndpointPool[] { }, ] if (provider === 'ollama') return pools - for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + for (const n of configuredPoolNumbers().usable) { const poolUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) const poolProvider = process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index a6e0a466c6..4198df93ab 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -15,9 +15,11 @@ import { endpointPoolProblems, finishDispatch, MAX_KEYS_PER_POOL, + MAX_POOL_NUMBER, missingProviderRefusal, POOL_KEY_STRIDE, prepareWorktree, + queenWorkerLimit, recordDispatch, resolveWorkerProvider, setDurableCloseListener, @@ -70,7 +72,18 @@ const KEYS = [ // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. 'TRIOS_QUEEN_WORKER_API_KEY_17', 'TRIOS_QUEEN_WORKER_API_KEY_1024', - 'TRIOS_QUEEN_WORKER_API_KEY_1025', + 'TRIOS_QUEEN_WORKER_API_KEY_4096', + // Names that look like key variables and are not: see the suffix rule. + 'TRIOS_QUEEN_WORKER_API_KEY_02', + 'TRIOS_QUEEN_WORKER_API_KEY_1', + 'TRIOS_QUEEN_WORKER_API_KEY_x', + // A ninth pool, and one whose number no durable index can address. + 'TRIOS_QUEEN_WORKER_POOL_9_BASE_URL', + 'TRIOS_QUEEN_WORKER_POOL_9_MODEL', + 'TRIOS_QUEEN_WORKER_POOL_9_API_KEY', + 'TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL', + 'TRIOS_QUEEN_WORKER_POOL_300000_MODEL', + 'TRIOS_QUEEN_WORKER_POOL_300000_API_KEY', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', ...POOL_VARIABLES, ] @@ -486,24 +499,86 @@ describe('queen dispatch precheck', () => { }) }) - it('reads a key past the sixteenth, up to the bound and not beyond it', () => { + it('reads every suffixed key that exists, in numeric order', () => { firstPool('z1') - // 16 was where the loop ended, not a measurement: a seventeenth variable - // was read by nothing and reported by nothing. + // The loop used to end at 16, then at 1024. Both were bounds that said + // nothing when they bound: the next variable was read by nothing. The + // names are now read from the environment, so there is no next variable. + process.env.TRIOS_QUEEN_WORKER_API_KEY_4096 = 'z4096' process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' process.env.TRIOS_QUEEN_WORKER_API_KEY_1024 = 'z1024' - process.env.TRIOS_QUEEN_WORKER_API_KEY_1025 = 'past-the-bound' process.env.TRIOS_QUEEN_MAX_WORKERS = '16' - expect(MAX_KEYS_PER_POOL).toBe(1024) expect(workerCapacityBreakdown()).toEqual({ - connectedCredentials: 3, + connectedCredentials: 4, lanesPerCredential: 1, - effectiveCapacity: 3, + effectiveCapacity: 4, }) + // Numeric, not lexical: 17 before 1024 before 4096. expect(resolveWorkerProvider([0])?.apiKey).toBe('z17') expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1024') - expect(resolveWorkerProvider([0, 1, 2])?.exhausted).toBe(3) + expect(resolveWorkerProvider([0, 1, 2])?.apiKey).toBe('z4096') + expect(resolveWorkerProvider([0, 1, 2, 3])?.exhausted).toBe(4) + }) + + it('ignores names that only look like key variables', () => { + firstPool('z1') + // `_1` would be a second name for the unsuffixed key, `_02` a second name + // for `_2`, `_x` no number at all. Guessing at any of them is how one + // secret becomes two slots. + process.env.TRIOS_QUEEN_WORKER_API_KEY_1 = 'not-a-slot' + process.env.TRIOS_QUEEN_WORKER_API_KEY_02 = 'not-a-slot-either' + process.env.TRIOS_QUEEN_WORKER_API_KEY_x = 'nor-this' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + }) + + it("cuts a pool where the next pool's durable index begins", () => { + firstPool('z1') + const extra = Array.from( + { length: MAX_KEYS_PER_POOL + 5 }, + (_, index) => `TRIOS_QUEEN_WORKER_API_KEY_${index + 2}`, + ) + for (const [index, name] of extra.entries()) { + process.env[name] = `z${index + 2}` + } + try { + // Not a policy: key_index of pool 2 starts at POOL_KEY_STRIDE, and that + // stride is already written in production rows. + expect(workerCapacityBreakdown().connectedCredentials).toBe( + MAX_KEYS_PER_POOL, + ) + expect(MAX_KEYS_PER_POOL).toBe(POOL_KEY_STRIDE - 1) + } finally { + for (const name of extra) delete process.env[name] + } + }) + + it('reads a ninth pool, because pools are discovered and not counted to eight', () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_9_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_9_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_9_API_KEY = 'n9' + + const choice = resolveWorkerProvider([0]) + expect(choice?.apiKey).toBe('n9') + expect(choice?.poolNumber).toBe(9) + expect(choice?.keyIndex).toBe(8 * POOL_KEY_STRIDE) + expect(endpointPoolProblems()).toEqual([]) + }) + + it('says so when a pool number cannot be named by a durable index', () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_300000_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_300000_API_KEY = 'far' + + expect(MAX_POOL_NUMBER).toBeLessThan(300000) + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(endpointPoolProblems().join(' ')).toContain( + 'TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL', + ) + expect(endpointPoolProblems().join(' ')).not.toContain('far') }) it('lets the credentials bind rather than a number nobody chose', () => { @@ -523,11 +598,11 @@ describe('queen dispatch precheck', () => { }) }) - it('still clamps a value that can only be a typo', () => { - // The clamp is not gone, it moved to where it can only catch a mistake: - // an operator who meant 50 and typed 5000 gets MAX_KEYS_PER_POOL bees, - // not fifty times the swarm. Enough credentials are connected that the - // ceiling, not the key list, is what answers. + it("lets the operator's number answer when the credentials are there to carry it", () => { + // This used to clamp at 1024. As a typo guard that protected little: an + // operator who meant 50 and typed 5000 over thousands of connected lanes + // got 1024 bees instead, and either number ends a container sized for + // fifty. What bounds a mistyped value is the credential list. firstPool('z1') const extra = Array.from( { length: MAX_KEYS_PER_POOL - 1 }, @@ -541,13 +616,24 @@ describe('queen dispatch precheck', () => { expect(workerCapacityBreakdown()).toEqual({ connectedCredentials: MAX_KEYS_PER_POOL, lanesPerCredential: 1, - effectiveCapacity: 1024, + effectiveCapacity: 5000, }) } finally { for (const name of extra) delete process.env[name] } }) + it('still refuses a value that is not a number of bees at all', () => { + process.env.TRIOS_QUEEN_MAX_WORKERS = '99999999999' + expect(queenWorkerLimit()).toBe(1_000_000) + process.env.TRIOS_QUEEN_MAX_WORKERS = 'many' + expect(queenWorkerLimit()).toBe(4) + process.env.TRIOS_QUEEN_MAX_WORKERS = '0' + expect(queenWorkerLimit()).toBe(4) + delete process.env.TRIOS_QUEEN_MAX_WORKERS + expect(queenWorkerLimit()).toBe(4) + }) + it('keeps a whole pool below the stride that separates pools', () => { // The durable index of pool n starts at (n - 1) * POOL_KEY_STRIDE. If a // pool could hold that many keys, its last key would BE pool n + 1's first. diff --git a/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift b/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift index 80dcedc237..e7a55e51d5 100644 --- a/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift +++ b/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift @@ -482,9 +482,14 @@ public enum QueenDelegationPolicy { /// Both sides now read one variable with one default and one ceiling, so /// the only way to raise the swarm is to raise it for both at once. /// - /// The ceiling here is 1024, and it is a guard against a typo rather than - /// a working value. It was 16, and 16 was the wrong KIND of number to - /// write into the policy: what a deployment can actually run is decided by + /// The ceiling here is 1_000_000, and it bounds no real deployment: it only + /// stops a value that is not a number of bees at all. It was 16 and then + /// 1024, and both were the wrong KIND of number to write into the policy. + /// As a typo guard 1024 protected little - an operator who meant 50 and + /// typed 5000 over two thousand connected lanes got 1024 bees instead of + /// 2000, and either ends a container sized for fifty. A mistyped value is + /// bounded by the credential list, because dispatch refuses when every + /// lane is taken. What a deployment can actually run is decided by /// how many credentials are connected times how many lanes each carries, /// and by the number the operator puts in TRIOS_QUEEN_MAX_WORKERS. Once a /// deployment could connect a second provider's endpoint and keys, a hard @@ -507,7 +512,7 @@ public enum QueenDelegationPolicy { public static var maximumConcurrentWorkers: Int { let raw = ProcessInfo.processInfo.environment["TRIOS_QUEEN_MAX_WORKERS"] guard let raw, let parsed = Int(raw), parsed >= 1 else { return 4 } - return min(parsed, 1024) + return min(parsed, 1_000_000) } public static func canStartAnother(running: Int) -> Bool { diff --git a/trios/rings/SR-00/QueenDelegation.swift b/trios/rings/SR-00/QueenDelegation.swift index 80dcedc237..e7a55e51d5 100644 --- a/trios/rings/SR-00/QueenDelegation.swift +++ b/trios/rings/SR-00/QueenDelegation.swift @@ -482,9 +482,14 @@ public enum QueenDelegationPolicy { /// Both sides now read one variable with one default and one ceiling, so /// the only way to raise the swarm is to raise it for both at once. /// - /// The ceiling here is 1024, and it is a guard against a typo rather than - /// a working value. It was 16, and 16 was the wrong KIND of number to - /// write into the policy: what a deployment can actually run is decided by + /// The ceiling here is 1_000_000, and it bounds no real deployment: it only + /// stops a value that is not a number of bees at all. It was 16 and then + /// 1024, and both were the wrong KIND of number to write into the policy. + /// As a typo guard 1024 protected little - an operator who meant 50 and + /// typed 5000 over two thousand connected lanes got 1024 bees instead of + /// 2000, and either ends a container sized for fifty. A mistyped value is + /// bounded by the credential list, because dispatch refuses when every + /// lane is taken. What a deployment can actually run is decided by /// how many credentials are connected times how many lanes each carries, /// and by the number the operator puts in TRIOS_QUEEN_MAX_WORKERS. Once a /// deployment could connect a second provider's endpoint and keys, a hard @@ -507,7 +512,7 @@ public enum QueenDelegationPolicy { public static var maximumConcurrentWorkers: Int { let raw = ProcessInfo.processInfo.environment["TRIOS_QUEEN_MAX_WORKERS"] guard let raw, let parsed = Int(raw), parsed >= 1 else { return 4 } - return min(parsed, 1024) + return min(parsed, 1_000_000) } public static func canStartAnother(running: Int) -> Bool {