diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 3423171b33..348e4c6693 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -232,23 +232,26 @@ export interface WorkerProvider { * its padding, and handing the trimmed form out keeps the count and the * selection - which both read this list - from ever disagreeing. * - * The suffix runs to MAX_KEYS_PER_POOL. It used to stop at 16, and 16 was never - * a measurement: it was where the loop happened to end, equal by coincidence to - * the policy ceiling on bees. They are different quantities. The ceiling bounds - * how many bees RUN; this bounds how many credentials the rotation may SPREAD - * them over, and a free key refuses its third concurrent request in under half - * a second (`1302`), so a swarm at the ceiling wants more keys than bees, not - * the same number. A seventeenth variable used to be read by nothing and - * reported by nothing, which is the zero-length-key trap again: it looks - * configured and supplies nothing. Widening the list does not widen the swarm - - * `queenWorkerLimit` still decides that. - * - * 1024 rather than a round thousand so that this bound can never sit BELOW a - * worker ceiling expressed as a power of two: with one lane per credential a - * swarm of N bees needs N keys, and a key bound under the worker bound would be - * a second, hidden ceiling that reports nothing when it is hit. + * The suffixes are READ FROM THE ENVIRONMENT, not counted up to a number. The + * loop used to stop at 16 and then at 1024, and both were the same kind of + * mistake: a bound that says nothing when it binds. `_17` was read by nothing + * while the loop ended at 16, and `_1025` would have been read by nothing + * after it. Asking the environment which suffixed names exist has no such + * edge - every variable an operator can see in the editor is a variable this + * function reads - and it costs the size of the environment rather than the + * size of a range. + * + * Order is the numeric suffix, the unsuffixed name first, so every index a + * deployment has already written still names the same key. A suffix must be a + * plain integer of two or more (`_2`, `_17`, `_4096`): `_02`, `_1` and `_x` + * are not key names and are ignored rather than guessed at. + * + * What remains is a consequence, not a policy: a pool is cut at + * MAX_KEYS_PER_POOL because the durable index of the next pool starts at + * POOL_KEY_STRIDE, and that stride is already written in production rows. + * Widening the list does not widen the swarm - `queenWorkerLimit` decides that. */ -export const MAX_KEYS_PER_POOL = 1024 +export const MAX_KEYS_PER_POOL = 9_999 function keysFor(envVar: string): string[] { const keys: string[] = [] @@ -260,8 +263,16 @@ function keysFor(envVar: string): string[] { keys.push(trimmed) } admit(process.env[envVar]) - for (let i = 2; i <= MAX_KEYS_PER_POOL; i++) { - admit(process.env[`${envVar}_${i}`]) + const prefix = `${envVar}_` + const suffixes = Object.keys(process.env) + .filter((name) => name.startsWith(prefix)) + .map((name) => name.slice(prefix.length)) + .filter((suffix) => /^[1-9]\d*$/.test(suffix) && Number(suffix) >= 2) + .map(Number) + .sort((a, b) => a - b) + for (const suffix of suffixes) { + if (keys.length >= MAX_KEYS_PER_POOL) break + admit(process.env[`${envVar}_${suffix}`]) } return keys } @@ -298,18 +309,25 @@ function workerLanesFor(provider: string): number { * dispatch, and promising less hides capacity that was paid for. * * So both sides now read the SAME environment variable, with the same default - * and the same ceiling. That ceiling is 1024 and it is a guard against a typo, - * not a working value: capacity is decided by connected credentials times - * lanes and by the number the operator sets. It used to be 16, which sat below - * the credentials a deployment could connect once endpoint pools existed, and - * said nothing when it bound. Review cost is still linear in running workers - - * that is why the operator picks the VALUE by measurement. The Swift side - * records the full reasoning. + * and the same ceiling. That ceiling is WORKER_SANITY_BOUND and it no longer + * bounds a real deployment: capacity is decided by connected credentials times + * lanes and by the number the operator sets. It was 16, then 1024, and both + * times it ended up sitting below credentials a deployment could connect and + * saying nothing when it bound. As a guard against a typo 1024 protected + * little: an operator who meant 50 and typed 5000 over two thousand connected + * lanes got 1024 bees instead of 2000, and either number ends a container + * sized for fifty. What actually bounds a mistyped value is the credential + * list - dispatch refuses when every lane is taken - so the constant only has + * to stop a value that is not a number of bees at all. Review cost is still + * linear in running workers, which is why the operator picks the VALUE by + * measurement. The Swift side records the full reasoning. */ -function queenWorkerLimit(): number { +export const WORKER_SANITY_BOUND = 1_000_000 + +export function queenWorkerLimit(): number { const parsed = Number(process.env.TRIOS_QUEEN_MAX_WORKERS) if (!Number.isInteger(parsed) || parsed < 1) return 4 - return Math.min(parsed, 1024) + return Math.min(parsed, WORKER_SANITY_BOUND) } /** @@ -502,13 +520,13 @@ function availableKeyIndex( * sixteen bees while three working NVIDIA keys sat unused next to them. * * So the FIRST pool is exactly the variables that already exist, unchanged in - * name and meaning, and further pools are numbered from 2: + * name and meaning, and further pools are numbered from 2, as many as exist: * * TRIOS_QUEEN_WORKER_POOL__BASE_URL the endpoint (required) * TRIOS_QUEEN_WORKER_POOL__MODEL the model THAT endpoint serves (required) * TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai * TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window - * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... (MAX_KEYS_PER_POOL) + * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2, _API_KEY_3 ... (any number of them) * * A pool is REMOTE credentials by definition. A local Ollama stays what it was * measured to be - one inference slot - and does not join a credential pool, @@ -521,7 +539,6 @@ function availableKeyIndex( * breakdown (#1308) an exact statement: credentials x lanes, bounded by the * policy ceiling. */ -const MAX_ENDPOINT_POOLS = 8 const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) /** @@ -539,6 +556,35 @@ const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) */ export const POOL_KEY_STRIDE = 10_000 +/** + * The numbers of the pools someone has started to configure, ascending. + * + * Discovered from the environment for the same reason the key suffixes are: + * "up to eight" was a bound nobody chose, and a ninth pool would have been + * configured in the editor and read by nothing. The only ceiling left is + * arithmetic - `key_index` is a 32-bit column and pool n starts at + * (n - 1) * POOL_KEY_STRIDE - so a pool number past MAX_POOL_NUMBER cannot be + * named durably and is reported by `endpointPoolProblems` instead of being + * silently dropped. + */ +export const MAX_POOL_NUMBER = Math.floor(2_147_483_647 / POOL_KEY_STRIDE) - 1 + +function configuredPoolNumbers(): { usable: number[]; tooLarge: number[] } { + const numbers = new Set() + for (const name of Object.keys(process.env)) { + const match = + /^TRIOS_QUEEN_WORKER_POOL_([1-9]\d*)_(BASE_URL|API_KEY(_[1-9]\d*)?)$/.exec( + name, + ) + if (match && process.env[name]?.trim()) numbers.add(Number(match[1])) + } + const sorted = [...numbers].filter((n) => n >= 2).sort((a, b) => a - b) + return { + usable: sorted.filter((n) => n <= MAX_POOL_NUMBER), + tooLarge: sorted.filter((n) => n > MAX_POOL_NUMBER), + } +} + interface EndpointPool { /** 1-based; 1 is the pool made of the unnumbered variables. */ number: number @@ -575,7 +621,13 @@ function contextWindowFrom(raw: string | undefined): number { */ export function endpointPoolProblems(): string[] { const problems: string[] = [] - for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const discovered = configuredPoolNumbers() + for (const n of discovered.tooLarge) { + problems.push( + `${poolVariable(n, 'BASE_URL')} names pool ${n}, above the ${MAX_POOL_NUMBER} a durable key index can address`, + ) + } + for (const n of discovered.usable) { const baseUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) const keys = keysFor(poolVariable(n, 'API_KEY')) if (!baseUrl && keys.length === 0) continue @@ -632,7 +684,7 @@ function configuredEndpointPools(): EndpointPool[] { }, ] if (provider === 'ollama') return pools - for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + for (const n of configuredPoolNumbers().usable) { const poolUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) const poolProvider = process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index a6e0a466c6..4198df93ab 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -15,9 +15,11 @@ import { endpointPoolProblems, finishDispatch, MAX_KEYS_PER_POOL, + MAX_POOL_NUMBER, missingProviderRefusal, POOL_KEY_STRIDE, prepareWorktree, + queenWorkerLimit, recordDispatch, resolveWorkerProvider, setDurableCloseListener, @@ -70,7 +72,18 @@ const KEYS = [ // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. 'TRIOS_QUEEN_WORKER_API_KEY_17', 'TRIOS_QUEEN_WORKER_API_KEY_1024', - 'TRIOS_QUEEN_WORKER_API_KEY_1025', + 'TRIOS_QUEEN_WORKER_API_KEY_4096', + // Names that look like key variables and are not: see the suffix rule. + 'TRIOS_QUEEN_WORKER_API_KEY_02', + 'TRIOS_QUEEN_WORKER_API_KEY_1', + 'TRIOS_QUEEN_WORKER_API_KEY_x', + // A ninth pool, and one whose number no durable index can address. + 'TRIOS_QUEEN_WORKER_POOL_9_BASE_URL', + 'TRIOS_QUEEN_WORKER_POOL_9_MODEL', + 'TRIOS_QUEEN_WORKER_POOL_9_API_KEY', + 'TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL', + 'TRIOS_QUEEN_WORKER_POOL_300000_MODEL', + 'TRIOS_QUEEN_WORKER_POOL_300000_API_KEY', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', ...POOL_VARIABLES, ] @@ -486,24 +499,86 @@ describe('queen dispatch precheck', () => { }) }) - it('reads a key past the sixteenth, up to the bound and not beyond it', () => { + it('reads every suffixed key that exists, in numeric order', () => { firstPool('z1') - // 16 was where the loop ended, not a measurement: a seventeenth variable - // was read by nothing and reported by nothing. + // The loop used to end at 16, then at 1024. Both were bounds that said + // nothing when they bound: the next variable was read by nothing. The + // names are now read from the environment, so there is no next variable. + process.env.TRIOS_QUEEN_WORKER_API_KEY_4096 = 'z4096' process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' process.env.TRIOS_QUEEN_WORKER_API_KEY_1024 = 'z1024' - process.env.TRIOS_QUEEN_WORKER_API_KEY_1025 = 'past-the-bound' process.env.TRIOS_QUEEN_MAX_WORKERS = '16' - expect(MAX_KEYS_PER_POOL).toBe(1024) expect(workerCapacityBreakdown()).toEqual({ - connectedCredentials: 3, + connectedCredentials: 4, lanesPerCredential: 1, - effectiveCapacity: 3, + effectiveCapacity: 4, }) + // Numeric, not lexical: 17 before 1024 before 4096. expect(resolveWorkerProvider([0])?.apiKey).toBe('z17') expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1024') - expect(resolveWorkerProvider([0, 1, 2])?.exhausted).toBe(3) + expect(resolveWorkerProvider([0, 1, 2])?.apiKey).toBe('z4096') + expect(resolveWorkerProvider([0, 1, 2, 3])?.exhausted).toBe(4) + }) + + it('ignores names that only look like key variables', () => { + firstPool('z1') + // `_1` would be a second name for the unsuffixed key, `_02` a second name + // for `_2`, `_x` no number at all. Guessing at any of them is how one + // secret becomes two slots. + process.env.TRIOS_QUEEN_WORKER_API_KEY_1 = 'not-a-slot' + process.env.TRIOS_QUEEN_WORKER_API_KEY_02 = 'not-a-slot-either' + process.env.TRIOS_QUEEN_WORKER_API_KEY_x = 'nor-this' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + }) + + it("cuts a pool where the next pool's durable index begins", () => { + firstPool('z1') + const extra = Array.from( + { length: MAX_KEYS_PER_POOL + 5 }, + (_, index) => `TRIOS_QUEEN_WORKER_API_KEY_${index + 2}`, + ) + for (const [index, name] of extra.entries()) { + process.env[name] = `z${index + 2}` + } + try { + // Not a policy: key_index of pool 2 starts at POOL_KEY_STRIDE, and that + // stride is already written in production rows. + expect(workerCapacityBreakdown().connectedCredentials).toBe( + MAX_KEYS_PER_POOL, + ) + expect(MAX_KEYS_PER_POOL).toBe(POOL_KEY_STRIDE - 1) + } finally { + for (const name of extra) delete process.env[name] + } + }) + + it('reads a ninth pool, because pools are discovered and not counted to eight', () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_9_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_9_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_9_API_KEY = 'n9' + + const choice = resolveWorkerProvider([0]) + expect(choice?.apiKey).toBe('n9') + expect(choice?.poolNumber).toBe(9) + expect(choice?.keyIndex).toBe(8 * POOL_KEY_STRIDE) + expect(endpointPoolProblems()).toEqual([]) + }) + + it('says so when a pool number cannot be named by a durable index', () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_300000_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_300000_API_KEY = 'far' + + expect(MAX_POOL_NUMBER).toBeLessThan(300000) + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(endpointPoolProblems().join(' ')).toContain( + 'TRIOS_QUEEN_WORKER_POOL_300000_BASE_URL', + ) + expect(endpointPoolProblems().join(' ')).not.toContain('far') }) it('lets the credentials bind rather than a number nobody chose', () => { @@ -523,11 +598,11 @@ describe('queen dispatch precheck', () => { }) }) - it('still clamps a value that can only be a typo', () => { - // The clamp is not gone, it moved to where it can only catch a mistake: - // an operator who meant 50 and typed 5000 gets MAX_KEYS_PER_POOL bees, - // not fifty times the swarm. Enough credentials are connected that the - // ceiling, not the key list, is what answers. + it("lets the operator's number answer when the credentials are there to carry it", () => { + // This used to clamp at 1024. As a typo guard that protected little: an + // operator who meant 50 and typed 5000 over thousands of connected lanes + // got 1024 bees instead, and either number ends a container sized for + // fifty. What bounds a mistyped value is the credential list. firstPool('z1') const extra = Array.from( { length: MAX_KEYS_PER_POOL - 1 }, @@ -541,13 +616,24 @@ describe('queen dispatch precheck', () => { expect(workerCapacityBreakdown()).toEqual({ connectedCredentials: MAX_KEYS_PER_POOL, lanesPerCredential: 1, - effectiveCapacity: 1024, + effectiveCapacity: 5000, }) } finally { for (const name of extra) delete process.env[name] } }) + it('still refuses a value that is not a number of bees at all', () => { + process.env.TRIOS_QUEEN_MAX_WORKERS = '99999999999' + expect(queenWorkerLimit()).toBe(1_000_000) + process.env.TRIOS_QUEEN_MAX_WORKERS = 'many' + expect(queenWorkerLimit()).toBe(4) + process.env.TRIOS_QUEEN_MAX_WORKERS = '0' + expect(queenWorkerLimit()).toBe(4) + delete process.env.TRIOS_QUEEN_MAX_WORKERS + expect(queenWorkerLimit()).toBe(4) + }) + it('keeps a whole pool below the stride that separates pools', () => { // The durable index of pool n starts at (n - 1) * POOL_KEY_STRIDE. If a // pool could hold that many keys, its last key would BE pool n + 1's first. diff --git a/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift b/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift index 80dcedc237..e7a55e51d5 100644 --- a/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift +++ b/trios/agent-server/queen-core/Sources/QueenPolicy/QueenDelegation.swift @@ -482,9 +482,14 @@ public enum QueenDelegationPolicy { /// Both sides now read one variable with one default and one ceiling, so /// the only way to raise the swarm is to raise it for both at once. /// - /// The ceiling here is 1024, and it is a guard against a typo rather than - /// a working value. It was 16, and 16 was the wrong KIND of number to - /// write into the policy: what a deployment can actually run is decided by + /// The ceiling here is 1_000_000, and it bounds no real deployment: it only + /// stops a value that is not a number of bees at all. It was 16 and then + /// 1024, and both were the wrong KIND of number to write into the policy. + /// As a typo guard 1024 protected little - an operator who meant 50 and + /// typed 5000 over two thousand connected lanes got 1024 bees instead of + /// 2000, and either ends a container sized for fifty. A mistyped value is + /// bounded by the credential list, because dispatch refuses when every + /// lane is taken. What a deployment can actually run is decided by /// how many credentials are connected times how many lanes each carries, /// and by the number the operator puts in TRIOS_QUEEN_MAX_WORKERS. Once a /// deployment could connect a second provider's endpoint and keys, a hard @@ -507,7 +512,7 @@ public enum QueenDelegationPolicy { public static var maximumConcurrentWorkers: Int { let raw = ProcessInfo.processInfo.environment["TRIOS_QUEEN_MAX_WORKERS"] guard let raw, let parsed = Int(raw), parsed >= 1 else { return 4 } - return min(parsed, 1024) + return min(parsed, 1_000_000) } public static func canStartAnother(running: Int) -> Bool { diff --git a/trios/rings/SR-00/QueenDelegation.swift b/trios/rings/SR-00/QueenDelegation.swift index 80dcedc237..e7a55e51d5 100644 --- a/trios/rings/SR-00/QueenDelegation.swift +++ b/trios/rings/SR-00/QueenDelegation.swift @@ -482,9 +482,14 @@ public enum QueenDelegationPolicy { /// Both sides now read one variable with one default and one ceiling, so /// the only way to raise the swarm is to raise it for both at once. /// - /// The ceiling here is 1024, and it is a guard against a typo rather than - /// a working value. It was 16, and 16 was the wrong KIND of number to - /// write into the policy: what a deployment can actually run is decided by + /// The ceiling here is 1_000_000, and it bounds no real deployment: it only + /// stops a value that is not a number of bees at all. It was 16 and then + /// 1024, and both were the wrong KIND of number to write into the policy. + /// As a typo guard 1024 protected little - an operator who meant 50 and + /// typed 5000 over two thousand connected lanes got 1024 bees instead of + /// 2000, and either ends a container sized for fifty. A mistyped value is + /// bounded by the credential list, because dispatch refuses when every + /// lane is taken. What a deployment can actually run is decided by /// how many credentials are connected times how many lanes each carries, /// and by the number the operator puts in TRIOS_QUEEN_MAX_WORKERS. Once a /// deployment could connect a second provider's endpoint and keys, a hard @@ -507,7 +512,7 @@ public enum QueenDelegationPolicy { public static var maximumConcurrentWorkers: Int { let raw = ProcessInfo.processInfo.environment["TRIOS_QUEEN_MAX_WORKERS"] guard let raw, let parsed = Int(raw), parsed >= 1 else { return 4 } - return min(parsed, 1024) + return min(parsed, 1_000_000) } public static func canStartAnother(running: Int) -> Bool {