diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 6de91ed17b..d934860bf8 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -163,9 +163,19 @@ export interface WorkerProvider { /** This server's own token, when the turn is aimed back at this server. */ apiKey?: string rehearsal?: boolean - /** Which of this provider's keys was handed out, 0-based. */ + /** + * Which key was handed out. For the first endpoint pool (and every legacy + * provider) this is the 0-based position in that pool's key list, exactly as + * before. A key of an ADDITIONAL endpoint pool carries its pool in the same + * integer - see POOL_KEY_STRIDE - because this is the value written to + * `queen_dispatch.key_index`, and one durable column has to name both. + */ keyIndex?: number + /** How many distinct keys the pool this key came from holds. */ keyCount?: number + /** 1-based number of the endpoint pool, present only when several are connected. */ + poolNumber?: number + poolCount?: number /** Which concurrent lane on this credential was handed out, 0-based. */ laneIndex?: number laneCount?: number @@ -221,7 +231,25 @@ export interface WorkerProvider { * trimmed value is also the one stored: a key that authenticates never needed * its padding, and handing the trimmed form out keeps the count and the * selection - which both read this list - from ever disagreeing. + * + * The suffix runs to MAX_KEYS_PER_POOL. It used to stop at 16, and 16 was never + * a measurement: it was where the loop happened to end, equal by coincidence to + * the policy ceiling on bees. They are different quantities. The ceiling bounds + * how many bees RUN; this bounds how many credentials the rotation may SPREAD + * them over, and a free key refuses its third concurrent request in under half + * a second (`1302`), so a swarm at the ceiling wants more keys than bees, not + * the same number. A seventeenth variable used to be read by nothing and + * reported by nothing, which is the zero-length-key trap again: it looks + * configured and supplies nothing. Widening the list does not widen the swarm - + * `queenWorkerLimit` still decides that. + * + * 1024 rather than a round thousand so that this bound can never sit BELOW a + * worker ceiling expressed as a power of two: with one lane per credential a + * swarm of N bees needs N keys, and a key bound under the worker bound would be + * a second, hidden ceiling that reports nothing when it is hit. */ +export const MAX_KEYS_PER_POOL = 1024 + function keysFor(envVar: string): string[] { const keys: string[] = [] const seen = new Set() @@ -232,7 +260,7 @@ function keysFor(envVar: string): string[] { keys.push(trimmed) } admit(process.env[envVar]) - for (let i = 2; i <= 16; i++) { + for (let i = 2; i <= MAX_KEYS_PER_POOL; i++) { admit(process.env[`${envVar}_${i}`]) } return keys @@ -326,16 +354,19 @@ function configuredRemoteLanesPerCredential(): number { export function workerCapacityBreakdown(): WorkerCapacityBreakdown { const endpoint = configuredWorkerBaseUrl() if (endpoint) { - const genericKeys = keysFor(GENERIC_WORKER_KEY_ENV) // An explicitly configured Ollama is one measured inference server even // when it happens to have an access token. Multiple names for that token // are not independent compute. A remote API, by contrast, gets exactly one - // conservative lane for every distinct credential. + // conservative lane for every distinct credential - of EVERY connected + // endpoint pool, because dispatch allocates against the same list. const provider = configuredWorkerProvider() const connectedCredentials = provider ? provider === 'ollama' ? 1 - : genericKeys.length + : configuredEndpointPools().reduce( + (total, pool) => total + pool.keys.length, + 0, + ) : 0 const lanesPerCredential = provider === 'ollama' ? 1 : configuredRemoteLanesPerCredential() @@ -455,6 +486,227 @@ function availableKeyIndex( return -1 } +/** + * More than one endpoint at once. + * + * One endpoint was the whole model: a single TRIOS_QUEEN_WORKER_BASE_URL, a + * single model, and every generic key sent there. A deployment that holds + * credentials for two providers could therefore use one of them - the second + * set of keys had nowhere to go, because a key handed to a bee is only as good + * as the URL the bee sends it to. Measured on 2026-09-17: ten Z.ai keys carried + * sixteen bees while three working NVIDIA keys sat unused next to them. + * + * So the FIRST pool is exactly the variables that already exist, unchanged in + * name and meaning, and further pools are numbered from 2: + * + * TRIOS_QUEEN_WORKER_POOL__BASE_URL the endpoint (required) + * TRIOS_QUEEN_WORKER_POOL__MODEL the model THAT endpoint serves (required) + * TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai + * TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window + * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... (MAX_KEYS_PER_POOL) + * + * A pool is REMOTE credentials by definition. A local Ollama stays what it was + * measured to be - one inference slot - and does not join a credential pool, + * so additional pools are read only when the first endpoint is a remote one. + * + * The model is required per pool and never inherited: the first pool's model + * name sent to a second provider is a 404 that arrives blamed on the work. + * Lanes per credential stay ONE number for the deployment + * (TRIOS_QUEEN_WORKER_LANES_PER_KEY), which keeps the closed capacity + * breakdown (#1308) an exact statement: credentials x lanes, bounded by the + * policy ceiling. + */ +const MAX_ENDPOINT_POOLS = 8 +const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) + +/** + * The durable name of a key in an additional pool. + * + * `queen_dispatch.key_index` is one integer, read back every tick to learn + * which credentials are busy. Keys of the first pool keep the index they have + * always had (0, 1, 2 ...), so rows written before this change still mean what + * they meant. A key of pool n is `(n - 1) * POOL_KEY_STRIDE + position`: pool + * 2's first key is 10000. The pool NUMBER is used, not its rank among the pools + * that happen to be valid today, so disconnecting pool 2 does not silently + * rename every busy key of pool 3. A pool holds at most MAX_KEYS_PER_POOL keys, + * which is below the stride, so the stride can never be reached from inside + * one - a test holds the two constants to that. + */ +export const POOL_KEY_STRIDE = 10_000 + +interface EndpointPool { + /** 1-based; 1 is the pool made of the unnumbered variables. */ + number: number + provider: string + baseUrl: string + model: string + contextWindow: number + keys: string[] +} + +function poolVariable(poolNumber: number, name: string): string { + return `TRIOS_QUEEN_WORKER_POOL_${poolNumber}_${name}` +} + +function cleanBaseUrl(raw: string | undefined): string | undefined { + return raw?.trim() ? raw.trim().replace(/\/+$/, '') : undefined +} + +function contextWindowFrom(raw: string | undefined): number { + const parsed = Number(raw) + if (!Number.isInteger(parsed) || parsed < 2048) { + return DEFAULT_LOCAL_CONTEXT_WINDOW + } + return parsed +} + +/** + * Why a numbered pool that someone started to configure is not connected. + * + * A pool with a URL and no model, or keys and an unsupported provider, reads as + * configured in a variable editor and supplies nothing - the same trap as an + * empty key box. Dispatch ignores such a pool rather than guessing, and this is + * the sentence that says so. Variable NAMES only; never a value. + */ +export function endpointPoolProblems(): string[] { + const problems: string[] = [] + for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const baseUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) + const keys = keysFor(poolVariable(n, 'API_KEY')) + if (!baseUrl && keys.length === 0) continue + if (!baseUrl) { + problems.push( + `${poolVariable(n, 'API_KEY')} is set but ${poolVariable(n, 'BASE_URL')} is not`, + ) + continue + } + const provider = + process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' + if (!REMOTE_ENDPOINT_PROVIDERS.has(provider)) { + problems.push( + `${poolVariable(n, 'PROVIDER')} must be one of ${[...REMOTE_ENDPOINT_PROVIDERS].join(', ')}`, + ) + } + if (!process.env[poolVariable(n, 'MODEL')]?.trim()) { + problems.push(`${poolVariable(n, 'MODEL')} is not set`) + } + if (keys.length === 0) { + problems.push(`${poolVariable(n, 'API_KEY')} is not set`) + } + } + return problems +} + +/** + * Every endpoint pool dispatch may hand a key from, first pool first. + * + * Empty when no endpoint is configured (the legacy provider path decides) or + * when the first endpoint's provider is unsupported - an explicit endpoint is + * authoritative, including its refusal, and a valid second pool must not + * quietly take over from a broken first one. + */ +function configuredEndpointPools(): EndpointPool[] { + const baseUrl = configuredWorkerBaseUrl() + const provider = configuredWorkerProvider() + if (!baseUrl || !provider) return [] + const seen = new Set() + const distinct = (keys: string[]) => + keys.filter((key) => { + if (seen.has(key)) return false + seen.add(key) + return true + }) + const pools: EndpointPool[] = [ + { + number: 1, + provider, + baseUrl, + model: process.env.TRIOS_QUEEN_WORKER_MODEL || DEFAULT_LOCAL_WORKER_MODEL, + contextWindow: configuredWorkerContextWindow(), + keys: distinct(keysFor(GENERIC_WORKER_KEY_ENV)), + }, + ] + if (provider === 'ollama') return pools + for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const poolUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) + const poolProvider = + process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' + const model = process.env[poolVariable(n, 'MODEL')]?.trim() + if (!poolUrl || !model || !REMOTE_ENDPOINT_PROVIDERS.has(poolProvider)) { + continue + } + // The same secret under two pools is still one account with one rate + // limit (#1293); the first pool that names it keeps it. + const keys = distinct(keysFor(poolVariable(n, 'API_KEY'))) + if (keys.length === 0) continue + pools.push({ + number: n, + provider: poolProvider, + baseUrl: poolUrl, + model, + contextWindow: contextWindowFrom(process.env[poolVariable(n, 'CONTEXT')]), + keys, + }) + } + return pools +} + +/** + * The least-used credential across every pool, scanning circularly after the + * last durable assignment. + * + * Every credential is one slot wherever it points, so a wave of bees spreads + * over both providers before any key carries a second lane - the same rule + * `availableKeyIndex` applies inside one pool, over the concatenated list. + */ +function endpointPoolProvider( + pools: EndpointPool[], + takenKeyIndices: number[], + afterKeyIndex?: number, +): WorkerProvider { + const slots = pools.flatMap((pool) => + pool.keys.map((key, position) => ({ + pool, + key, + durableIndex: (pool.number - 1) * POOL_KEY_STRIDE + position, + })), + ) + const laneCount = configuredRemoteLanesPerCredential() + const occupancy = slots.map( + (slot) => + takenKeyIndices.filter((taken) => taken === slot.durableIndex).length, + ) + // The cursor is a durable index, not a position: translate it, and start + // from the top when it names a key that is no longer connected. + const cursor = slots.findIndex((slot) => slot.durableIndex === afterKeyIndex) + const position = availableKeyIndex( + occupancy, + laneCount, + cursor >= 0 ? cursor : undefined, + ) + if (position < 0) { + return { + provider: pools[0].provider, + model: pools[0].model, + exhausted: slots.length * laneCount, + } + } + const { pool, key, durableIndex } = slots[position] + return { + provider: pool.provider, + model: pool.model, + baseUrl: pool.baseUrl, + apiKey: key, + keyIndex: durableIndex, + keyCount: pool.keys.length, + poolNumber: pool.number, + poolCount: pools.length, + laneIndex: occupancy[position], + laneCount, + contextWindow: pool.contextWindow, + } +} + function configuredEndpointProvider( override: string | undefined, takenKeyIndices: number[], @@ -462,6 +714,13 @@ function configuredEndpointProvider( ): WorkerProvider | null { const baseUrl = configuredWorkerBaseUrl() if (!baseUrl) return null + // Several connected pools: one allocator over all of them. A single pool + // takes the path below, byte for byte what it was. + const pools = configuredEndpointPools() + if (pools.length > 1) { + if (pools[0].keys.length === 0) return null + return endpointPoolProvider(pools, takenKeyIndices, afterKeyIndex) + } const model = override || DEFAULT_LOCAL_WORKER_MODEL const provider = configuredWorkerProvider() if (!provider) return null @@ -2542,9 +2801,15 @@ export async function dispatchBee( const nextKey = keyVariable ? `${keyVariable}_${chosen.exhausted + 1}` : 'the matching provider variable' + // A numbered pool someone began to configure and dispatch ignored is the + // likeliest reason the swarm is narrower than the operator believes. + const ignoredPools = configuredWorkerBaseUrl() ? endpointPoolProblems() : [] const detail = `all ${chosen.exhausted} provider key(s) are already in use by bees in ` + - `flight. Add another with ${nextKey} to widen the swarm.` + `flight. Add another with ${nextKey} to widen the swarm.` + + (ignoredPools.length > 0 + ? ` Not connected: ${ignoredPools.join('; ')}.` + : '') logger.warn('Queen tick chose an issue but every key is busy', { issue, detail, @@ -2605,8 +2870,11 @@ export async function dispatchBee( ) const detail = turn.ok ? `${worktree.detail}; ${chosen.provider}/${chosen.model}` + + (chosen.poolCount && chosen.poolCount > 1 + ? ` pool ${chosen.poolNumber}` + : '') + (chosen.keyCount && chosen.keyCount > 1 - ? ` key ${(chosen.keyIndex ?? 0) + 1}/${chosen.keyCount}` + ? ` key ${((chosen.keyIndex ?? 0) % POOL_KEY_STRIDE) + 1}/${chosen.keyCount}` : '') + (chosen.laneCount && chosen.laneCount > 1 ? ` lane ${(chosen.laneIndex ?? 0) + 1}/${chosen.laneCount}` diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index e455c6b4f5..bae5d90f94 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -12,8 +12,11 @@ import { configuredWorkerLanesPerCredential, dispatchBee, drain, + endpointPoolProblems, finishDispatch, + MAX_KEYS_PER_POOL, missingProviderRefusal, + POOL_KEY_STRIDE, prepareWorktree, recordDispatch, resolveWorkerProvider, @@ -31,6 +34,18 @@ const GENERIC_WORKER_KEYS = [ ), ] +// Additional endpoint pools (TRIOS_QUEEN_WORKER_POOL__*). Listed for the same +// reason as every other name here: one process runs every api test file, and a +// pool left behind makes the next file's "one endpoint" case read two. +const POOL_VARIABLES = [2, 3].flatMap((pool) => [ + ...['BASE_URL', 'MODEL', 'PROVIDER', 'CONTEXT', 'API_KEY'].map( + (name) => `TRIOS_QUEEN_WORKER_POOL_${pool}_${name}`, + ), + ...[2, 3, 4].map( + (index) => `TRIOS_QUEEN_WORKER_POOL_${pool}_API_KEY_${index}`, + ), +]) + const KEYS = [ 'ZAI_API_KEY', 'ZAI_API_KEY_2', @@ -50,7 +65,14 @@ const KEYS = [ 'TRIOS_QUEEN_WORKER_BASE_URL', 'TRIOS_QUEEN_WORKER_MODEL', 'TRIOS_QUEEN_WORKER_CONTEXT', + 'TRIOS_QUEEN_WORKER_LANES_PER_KEY', + 'TRIOS_QUEEN_MAX_WORKERS', + // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. + 'TRIOS_QUEEN_WORKER_API_KEY_17', + 'TRIOS_QUEEN_WORKER_API_KEY_1024', + 'TRIOS_QUEEN_WORKER_API_KEY_1025', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', + ...POOL_VARIABLES, ] afterEach(() => { @@ -240,6 +262,299 @@ describe('queen dispatch precheck', () => { }) }) + /** + * Measured on 2026-09-17: the deployment held ten working Z.ai keys and three + * working NVIDIA keys, and only the first ten could carry a bee, because one + * endpoint URL was the whole model and a key is only as good as the URL the + * bee sends it to. A numbered pool is a second endpoint with its own model + * and its own keys; the first pool is the unnumbered variables, unchanged. + */ + describe('additional endpoint pools', () => { + const ZAI_URL = 'https://api.z.ai/api/paas/v4' + const NVIDIA_URL = 'https://integrate.api.nvidia.com/v1' + const NVIDIA_MODEL = 'nvidia/nemotron-3-super-120b-a12b' + + const firstPool = (...keys: string[]) => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = ZAI_URL + process.env.TRIOS_QUEEN_WORKER_MODEL = 'glm-4.5-flash' + process.env.TRIOS_QUEEN_WORKER_CONTEXT = '65536' + keys.forEach((value, index) => { + process.env[GENERIC_WORKER_KEYS[index]] = value + }) + } + const secondPool = (...keys: string[]) => { + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = `${NVIDIA_URL}/` + process.env.TRIOS_QUEEN_WORKER_POOL_2_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_2_CONTEXT = '131072' + keys.forEach((value, index) => { + const name = + index === 0 + ? 'TRIOS_QUEEN_WORKER_POOL_2_API_KEY' + : `TRIOS_QUEEN_WORKER_POOL_2_API_KEY_${index + 1}` + process.env[name] = value + }) + } + + it('sends each key to the endpoint of its own pool', () => { + firstPool('z1', 'z2', 'z3') + secondPool('n1', 'n2') + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 5, + lanesPerCredential: 1, + effectiveCapacity: 5, + }) + const taken: number[] = [] + const choices = Array.from({ length: 5 }, () => { + const choice = resolveWorkerProvider([...taken]) + taken.push(choice?.keyIndex ?? -1) + return choice + }) + // The first pool keeps the indices it has always had; pool 2 starts at + // the stride, so rows written before this change still mean the same key. + expect(choices.map((choice) => choice?.keyIndex)).toEqual([ + 0, + 1, + 2, + POOL_KEY_STRIDE, + POOL_KEY_STRIDE + 1, + ]) + expect(choices.map((choice) => choice?.apiKey)).toEqual([ + 'z1', + 'z2', + 'z3', + 'n1', + 'n2', + ]) + expect(choices[0]).toMatchObject({ + provider: 'zai', + baseUrl: ZAI_URL, + model: 'glm-4.5-flash', + contextWindow: 65536, + keyCount: 3, + poolNumber: 1, + poolCount: 2, + }) + // A Z.ai URL with an NVIDIA key is a 401 blamed on the work. Everything + // that travels to /chat comes from the SAME pool as the key. + expect(choices[3]).toMatchObject({ + provider: 'openai-compatible', + baseUrl: NVIDIA_URL, + model: NVIDIA_MODEL, + contextWindow: 131072, + keyCount: 2, + poolNumber: 2, + poolCount: 2, + }) + expect(resolveWorkerProvider(taken)?.exhausted).toBe(5) + }) + + it('rotates across pools after the last durable assignment', () => { + firstPool('z1', 'z2') + secondPool('n1', 'n2') + + expect(resolveWorkerProvider([], 1)?.keyIndex).toBe(POOL_KEY_STRIDE) + expect(resolveWorkerProvider([], POOL_KEY_STRIDE)?.keyIndex).toBe( + POOL_KEY_STRIDE + 1, + ) + expect(resolveWorkerProvider([], POOL_KEY_STRIDE + 1)?.keyIndex).toBe(0) + // A cursor naming a key that is no longer connected starts from the top + // instead of being read as a position in a list it was never part of. + expect(resolveWorkerProvider([], 2 * POOL_KEY_STRIDE + 7)?.keyIndex).toBe( + 0, + ) + }) + + it('gives every credential of every pool one bee before any key carries two', () => { + firstPool('z1', 'z2') + secondPool('n1') + process.env.TRIOS_QUEEN_WORKER_LANES_PER_KEY = '2' + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 3, + lanesPerCredential: 2, + effectiveCapacity: 6, + }) + const taken: number[] = [] + for (let bee = 0; bee < 6; bee++) { + const choice = resolveWorkerProvider( + [...taken], + taken[taken.length - 1], + ) + taken.push(choice?.keyIndex ?? -1) + } + expect(taken).toEqual([0, 1, POOL_KEY_STRIDE, 0, 1, POOL_KEY_STRIDE]) + expect(resolveWorkerProvider(taken)?.exhausted).toBe(6) + }) + + it('stays behind the policy ceiling however many pools are connected', () => { + firstPool('z1', 'z2', 'z3') + secondPool('n1', 'n2', 'n3') + + // TRIOS_QUEEN_MAX_WORKERS unset: the measured default of four. + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 6, + lanesPerCredential: 1, + effectiveCapacity: 4, + }) + expect(configuredWorkerCapacity()).toBe(4) + }) + + it('ignores a half-configured pool and says which variable is missing', () => { + firstPool('z1') + // A URL and keys but no model: the first pool's model name sent to + // another provider is a 404, so the pool is not connected at all. + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_2_API_KEY = 'n1' + // Keys with no endpoint to send them to. + process.env.TRIOS_QUEEN_WORKER_POOL_3_API_KEY = 'x1' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(resolveWorkerProvider([0])?.exhausted).toBe(1) + const problems = endpointPoolProblems() + expect(problems).toContain('TRIOS_QUEEN_WORKER_POOL_2_MODEL is not set') + expect(problems.join(' ')).toContain('TRIOS_QUEEN_WORKER_POOL_3_BASE_URL') + // Names only. A value in a diagnostic is a disclosure. + expect(problems.join(' ')).not.toContain('n1') + expect(problems.join(' ')).not.toContain('x1') + }) + + it('names an ignored pool when every connected key is busy', async () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_2_API_KEY = 'n1' + const pool = { + query: async () => ({ rowCount: 1, rows: [] }), + } as unknown as Pool + + const outcome = await dispatchBee(pool, 1308, 'brief', [], [0]) + + expect(outcome.started).toBe(false) + expect(outcome.detail).toContain('TRIOS_QUEEN_WORKER_API_KEY_2') + expect(outcome.detail).toContain( + 'TRIOS_QUEEN_WORKER_POOL_2_MODEL is not set', + ) + expect(outcome.detail).not.toContain('n1') + }) + + it('refuses a pool whose provider is not a remote endpoint', () => { + firstPool('z1') + secondPool('n1') + process.env.TRIOS_QUEEN_WORKER_POOL_2_PROVIDER = 'ollama' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(endpointPoolProblems().join(' ')).toContain( + 'TRIOS_QUEEN_WORKER_POOL_2_PROVIDER', + ) + }) + + it('counts one secret named by two pools as one credential', () => { + firstPool('shared', 'z2') + secondPool('shared', 'n2') + + expect(workerCapacityBreakdown().connectedCredentials).toBe(3) + const second = resolveWorkerProvider([0, 1]) + // The first pool that names a secret keeps it; pool 2 offers only 'n2'. + expect(second?.apiKey).toBe('n2') + expect(second?.keyIndex).toBe(POOL_KEY_STRIDE) + expect(second?.keyCount).toBe(1) + }) + + it('does not let a valid second pool take over from a first one with no key', () => { + firstPool() + secondPool('n1') + + // An explicit endpoint is authoritative, including its refusal. + expect(resolveWorkerProvider()).toBeNull() + expect(missingProviderRefusal()).toContain('TRIOS_QUEEN_WORKER_API_KEY') + }) + + it('keeps a local first endpoint as one measured lane and reads no pools', () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'ollama' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'http://ollama:11434/v1' + secondPool('n1', 'n2') + + expect(resolveWorkerProvider()?.provider).toBe('ollama') + expect(resolveWorkerProvider()?.poolCount).toBeUndefined() + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 1, + lanesPerCredential: 1, + effectiveCapacity: 1, + }) + }) + + it('reads a key past the sixteenth, up to the bound and not beyond it', () => { + firstPool('z1') + // 16 was where the loop ended, not a measurement: a seventeenth variable + // was read by nothing and reported by nothing. + process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1024 = 'z1024' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1025 = 'past-the-bound' + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(MAX_KEYS_PER_POOL).toBe(1024) + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 3, + lanesPerCredential: 1, + effectiveCapacity: 3, + }) + expect(resolveWorkerProvider([0])?.apiKey).toBe('z17') + expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1024') + expect(resolveWorkerProvider([0, 1, 2])?.exhausted).toBe(3) + }) + + it('widens the key list without widening the swarm', () => { + firstPool(...Array.from({ length: 16 }, (_, index) => `z${index + 1}`)) + process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' + process.env.TRIOS_QUEEN_MAX_WORKERS = '64' + + // Seventeen credentials, and the policy ceiling still answers sixteen. + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 17, + lanesPerCredential: 1, + effectiveCapacity: 16, + }) + }) + + it('keeps a whole pool below the stride that separates pools', () => { + // The durable index of pool n starts at (n - 1) * POOL_KEY_STRIDE. If a + // pool could hold that many keys, its last key would BE pool n + 1's first. + expect(MAX_KEYS_PER_POOL).toBeLessThan(POOL_KEY_STRIDE) + }) + + it('leaves a single endpoint exactly as it was', () => { + firstPool('z1', 'z2') + + const choice = resolveWorkerProvider([0]) + expect(choice?.keyIndex).toBe(1) + expect(choice?.poolNumber).toBeUndefined() + expect(choice?.poolCount).toBeUndefined() + expect(endpointPoolProblems()).toEqual([]) + }) + + it('keeps the breakdown closed when several pools are connected', () => { + firstPool('planted-secret-z') + secondPool('planted-secret-n') + const breakdown = workerCapacityBreakdown() as unknown as Record< + string, + unknown + > + + expect(Object.keys(breakdown).sort()).toEqual([ + 'connectedCredentials', + 'effectiveCapacity', + 'lanesPerCredential', + ]) + const serialized = JSON.stringify(breakdown) + expect(serialized).not.toContain('planted-secret') + expect(serialized).not.toContain('POOL_2') + expect(serialized).not.toContain('nvidia') + }) + }) + // Four bees on one key share one rate limit, so the swarm's real ceiling // becomes whatever that key allows rather than what the Queen permits - and // the 429 arrives blamed on the work.