From d77234a0aab67fc83b1a68690cf9bfb11f0db676 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 17 Sep 2026 17:16:24 +0700 Subject: [PATCH 1/3] feat(queen): a second provider's keys had nowhere to go, because one URL was the whole pool Measured on 2026-09-17 in production: ten working Z.ai keys carried sixteen bees while three working NVIDIA keys sat unused beside them. A key handed to a bee is only as good as the URL the bee sends it to, and the deployment has exactly one: TRIOS_QUEEN_WORKER_BASE_URL, one model, every generic key sent there. Putting an NVIDIA key into TRIOS_QUEEN_WORKER_API_KEY_n would not widen the swarm, it would hand a bee a 401 that arrives blamed on the work. So there are now numbered endpoint pools. The FIRST pool is the variables that already exist, unchanged in name and meaning. Further pools start at 2: TRIOS_QUEEN_WORKER_POOL__BASE_URL the endpoint (required) TRIOS_QUEEN_WORKER_POOL__MODEL the model that endpoint serves (required) TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... _API_KEY_16 What travels to /chat - provider, base URL, model, context window - comes from the same pool as the key. One allocator runs over the concatenated list, so a wave of bees takes one credential of every pool before any key carries a second lane: the rule availableKeyIndex already applied inside one pool. queen_dispatch.key_index is one integer, read back every tick to learn which credentials are busy. First-pool keys keep the index they have always had, so rows in flight across the deploy still mean the same key. A key of pool n is (n - 1) * POOL_KEY_STRIDE + position (pool 2 starts at 1000). The pool NUMBER is used rather than its rank among today's valid pools, so disconnecting pool 2 does not rename every busy key of pool 3. Deliberately not changed: - A single endpoint takes the old code path byte for byte; every existing test passes untouched (66 before, 78 after). - The closed breakdown (#1308) stays three integers. Lanes per credential stay one number for the deployment, which keeps it an exact statement: credentials x lanes, bounded by the policy ceiling. No pool name, URL or variable name enters it. - An explicit endpoint stays authoritative, including its refusal: a valid second pool does not take over from a first pool with no key. - A local Ollama stays one measured inference slot and reads no pools. - The model is required per pool and never inherited: the first pool's model name sent to another provider is a 404. - One secret named by two pools is one credential (#1293). A pool someone began to configure and dispatch ignored reads as configured in a variable editor and supplies nothing. endpointPoolProblems() names the missing variable (names only, never a value), and the "every key is busy" refusal carries it, because that is the moment an operator asks why the swarm is narrower than the variables suggest. The dispatch detail printed `key ${keyIndex + 1}`; with a durable index of 1002 that would read "key 1003/3". It now prints the pool and the position in it. Applies cleanly to fix/queen-worker-provider-and-prompt-size (the branch production deploys from) as well as to feat/queen-supervisor. Co-Authored-By: Claude Fable 5.1 --- .../server/src/api/services/queen-dispatch.ts | 261 ++++++++++++++++- .../server/tests/api/queen-dispatch.test.ts | 271 ++++++++++++++++++ 2 files changed, 526 insertions(+), 6 deletions(-) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 6de91ed17b..95028b17e7 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -163,9 +163,19 @@ export interface WorkerProvider { /** This server's own token, when the turn is aimed back at this server. */ apiKey?: string rehearsal?: boolean - /** Which of this provider's keys was handed out, 0-based. */ + /** + * Which key was handed out. For the first endpoint pool (and every legacy + * provider) this is the 0-based position in that pool's key list, exactly as + * before. A key of an ADDITIONAL endpoint pool carries its pool in the same + * integer - see POOL_KEY_STRIDE - because this is the value written to + * `queen_dispatch.key_index`, and one durable column has to name both. + */ keyIndex?: number + /** How many distinct keys the pool this key came from holds. */ keyCount?: number + /** 1-based number of the endpoint pool, present only when several are connected. */ + poolNumber?: number + poolCount?: number /** Which concurrent lane on this credential was handed out, 0-based. */ laneIndex?: number laneCount?: number @@ -326,16 +336,19 @@ function configuredRemoteLanesPerCredential(): number { export function workerCapacityBreakdown(): WorkerCapacityBreakdown { const endpoint = configuredWorkerBaseUrl() if (endpoint) { - const genericKeys = keysFor(GENERIC_WORKER_KEY_ENV) // An explicitly configured Ollama is one measured inference server even // when it happens to have an access token. Multiple names for that token // are not independent compute. A remote API, by contrast, gets exactly one - // conservative lane for every distinct credential. + // conservative lane for every distinct credential - of EVERY connected + // endpoint pool, because dispatch allocates against the same list. const provider = configuredWorkerProvider() const connectedCredentials = provider ? provider === 'ollama' ? 1 - : genericKeys.length + : configuredEndpointPools().reduce( + (total, pool) => total + pool.keys.length, + 0, + ) : 0 const lanesPerCredential = provider === 'ollama' ? 1 : configuredRemoteLanesPerCredential() @@ -455,6 +468,226 @@ function availableKeyIndex( return -1 } +/** + * More than one endpoint at once. + * + * One endpoint was the whole model: a single TRIOS_QUEEN_WORKER_BASE_URL, a + * single model, and every generic key sent there. A deployment that holds + * credentials for two providers could therefore use one of them - the second + * set of keys had nowhere to go, because a key handed to a bee is only as good + * as the URL the bee sends it to. Measured on 2026-09-17: ten Z.ai keys carried + * sixteen bees while three working NVIDIA keys sat unused next to them. + * + * So the FIRST pool is exactly the variables that already exist, unchanged in + * name and meaning, and further pools are numbered from 2: + * + * TRIOS_QUEEN_WORKER_POOL__BASE_URL the endpoint (required) + * TRIOS_QUEEN_WORKER_POOL__MODEL the model THAT endpoint serves (required) + * TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai + * TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window + * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... _API_KEY_16 + * + * A pool is REMOTE credentials by definition. A local Ollama stays what it was + * measured to be - one inference slot - and does not join a credential pool, + * so additional pools are read only when the first endpoint is a remote one. + * + * The model is required per pool and never inherited: the first pool's model + * name sent to a second provider is a 404 that arrives blamed on the work. + * Lanes per credential stay ONE number for the deployment + * (TRIOS_QUEEN_WORKER_LANES_PER_KEY), which keeps the closed capacity + * breakdown (#1308) an exact statement: credentials x lanes, bounded by the + * policy ceiling. + */ +const MAX_ENDPOINT_POOLS = 8 +const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) + +/** + * The durable name of a key in an additional pool. + * + * `queen_dispatch.key_index` is one integer, read back every tick to learn + * which credentials are busy. Keys of the first pool keep the index they have + * always had (0, 1, 2 ...), so rows written before this change still mean what + * they meant. A key of pool n is `(n - 1) * POOL_KEY_STRIDE + position`: pool + * 2's first key is 1000. The pool NUMBER is used, not its rank among the pools + * that happen to be valid today, so disconnecting pool 2 does not silently + * rename every busy key of pool 3. A pool holds at most 16 keys, so the stride + * can never be reached from inside one. + */ +export const POOL_KEY_STRIDE = 1000 + +interface EndpointPool { + /** 1-based; 1 is the pool made of the unnumbered variables. */ + number: number + provider: string + baseUrl: string + model: string + contextWindow: number + keys: string[] +} + +function poolVariable(poolNumber: number, name: string): string { + return `TRIOS_QUEEN_WORKER_POOL_${poolNumber}_${name}` +} + +function cleanBaseUrl(raw: string | undefined): string | undefined { + return raw?.trim() ? raw.trim().replace(/\/+$/, '') : undefined +} + +function contextWindowFrom(raw: string | undefined): number { + const parsed = Number(raw) + if (!Number.isInteger(parsed) || parsed < 2048) { + return DEFAULT_LOCAL_CONTEXT_WINDOW + } + return parsed +} + +/** + * Why a numbered pool that someone started to configure is not connected. + * + * A pool with a URL and no model, or keys and an unsupported provider, reads as + * configured in a variable editor and supplies nothing - the same trap as an + * empty key box. Dispatch ignores such a pool rather than guessing, and this is + * the sentence that says so. Variable NAMES only; never a value. + */ +export function endpointPoolProblems(): string[] { + const problems: string[] = [] + for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const baseUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) + const keys = keysFor(poolVariable(n, 'API_KEY')) + if (!baseUrl && keys.length === 0) continue + if (!baseUrl) { + problems.push( + `${poolVariable(n, 'API_KEY')} is set but ${poolVariable(n, 'BASE_URL')} is not`, + ) + continue + } + const provider = + process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' + if (!REMOTE_ENDPOINT_PROVIDERS.has(provider)) { + problems.push( + `${poolVariable(n, 'PROVIDER')} must be one of ${[...REMOTE_ENDPOINT_PROVIDERS].join(', ')}`, + ) + } + if (!process.env[poolVariable(n, 'MODEL')]?.trim()) { + problems.push(`${poolVariable(n, 'MODEL')} is not set`) + } + if (keys.length === 0) { + problems.push(`${poolVariable(n, 'API_KEY')} is not set`) + } + } + return problems +} + +/** + * Every endpoint pool dispatch may hand a key from, first pool first. + * + * Empty when no endpoint is configured (the legacy provider path decides) or + * when the first endpoint's provider is unsupported - an explicit endpoint is + * authoritative, including its refusal, and a valid second pool must not + * quietly take over from a broken first one. + */ +function configuredEndpointPools(): EndpointPool[] { + const baseUrl = configuredWorkerBaseUrl() + const provider = configuredWorkerProvider() + if (!baseUrl || !provider) return [] + const seen = new Set() + const distinct = (keys: string[]) => + keys.filter((key) => { + if (seen.has(key)) return false + seen.add(key) + return true + }) + const pools: EndpointPool[] = [ + { + number: 1, + provider, + baseUrl, + model: process.env.TRIOS_QUEEN_WORKER_MODEL || DEFAULT_LOCAL_WORKER_MODEL, + contextWindow: configuredWorkerContextWindow(), + keys: distinct(keysFor(GENERIC_WORKER_KEY_ENV)), + }, + ] + if (provider === 'ollama') return pools + for (let n = 2; n <= MAX_ENDPOINT_POOLS; n++) { + const poolUrl = cleanBaseUrl(process.env[poolVariable(n, 'BASE_URL')]) + const poolProvider = + process.env[poolVariable(n, 'PROVIDER')]?.trim() || 'openai-compatible' + const model = process.env[poolVariable(n, 'MODEL')]?.trim() + if (!poolUrl || !model || !REMOTE_ENDPOINT_PROVIDERS.has(poolProvider)) { + continue + } + // The same secret under two pools is still one account with one rate + // limit (#1293); the first pool that names it keeps it. + const keys = distinct(keysFor(poolVariable(n, 'API_KEY'))) + if (keys.length === 0) continue + pools.push({ + number: n, + provider: poolProvider, + baseUrl: poolUrl, + model, + contextWindow: contextWindowFrom(process.env[poolVariable(n, 'CONTEXT')]), + keys, + }) + } + return pools +} + +/** + * The least-used credential across every pool, scanning circularly after the + * last durable assignment. + * + * Every credential is one slot wherever it points, so a wave of bees spreads + * over both providers before any key carries a second lane - the same rule + * `availableKeyIndex` applies inside one pool, over the concatenated list. + */ +function endpointPoolProvider( + pools: EndpointPool[], + takenKeyIndices: number[], + afterKeyIndex?: number, +): WorkerProvider { + const slots = pools.flatMap((pool) => + pool.keys.map((key, position) => ({ + pool, + key, + durableIndex: (pool.number - 1) * POOL_KEY_STRIDE + position, + })), + ) + const laneCount = configuredRemoteLanesPerCredential() + const occupancy = slots.map( + (slot) => + takenKeyIndices.filter((taken) => taken === slot.durableIndex).length, + ) + // The cursor is a durable index, not a position: translate it, and start + // from the top when it names a key that is no longer connected. + const cursor = slots.findIndex((slot) => slot.durableIndex === afterKeyIndex) + const position = availableKeyIndex( + occupancy, + laneCount, + cursor >= 0 ? cursor : undefined, + ) + if (position < 0) { + return { + provider: pools[0].provider, + model: pools[0].model, + exhausted: slots.length * laneCount, + } + } + const { pool, key, durableIndex } = slots[position] + return { + provider: pool.provider, + model: pool.model, + baseUrl: pool.baseUrl, + apiKey: key, + keyIndex: durableIndex, + keyCount: pool.keys.length, + poolNumber: pool.number, + poolCount: pools.length, + laneIndex: occupancy[position], + laneCount, + contextWindow: pool.contextWindow, + } +} + function configuredEndpointProvider( override: string | undefined, takenKeyIndices: number[], @@ -462,6 +695,13 @@ function configuredEndpointProvider( ): WorkerProvider | null { const baseUrl = configuredWorkerBaseUrl() if (!baseUrl) return null + // Several connected pools: one allocator over all of them. A single pool + // takes the path below, byte for byte what it was. + const pools = configuredEndpointPools() + if (pools.length > 1) { + if (pools[0].keys.length === 0) return null + return endpointPoolProvider(pools, takenKeyIndices, afterKeyIndex) + } const model = override || DEFAULT_LOCAL_WORKER_MODEL const provider = configuredWorkerProvider() if (!provider) return null @@ -2542,9 +2782,15 @@ export async function dispatchBee( const nextKey = keyVariable ? `${keyVariable}_${chosen.exhausted + 1}` : 'the matching provider variable' + // A numbered pool someone began to configure and dispatch ignored is the + // likeliest reason the swarm is narrower than the operator believes. + const ignoredPools = configuredWorkerBaseUrl() ? endpointPoolProblems() : [] const detail = `all ${chosen.exhausted} provider key(s) are already in use by bees in ` + - `flight. Add another with ${nextKey} to widen the swarm.` + `flight. Add another with ${nextKey} to widen the swarm.` + + (ignoredPools.length > 0 + ? ` Not connected: ${ignoredPools.join('; ')}.` + : '') logger.warn('Queen tick chose an issue but every key is busy', { issue, detail, @@ -2605,8 +2851,11 @@ export async function dispatchBee( ) const detail = turn.ok ? `${worktree.detail}; ${chosen.provider}/${chosen.model}` + + (chosen.poolCount && chosen.poolCount > 1 + ? ` pool ${chosen.poolNumber}` + : '') + (chosen.keyCount && chosen.keyCount > 1 - ? ` key ${(chosen.keyIndex ?? 0) + 1}/${chosen.keyCount}` + ? ` key ${((chosen.keyIndex ?? 0) % POOL_KEY_STRIDE) + 1}/${chosen.keyCount}` : '') + (chosen.laneCount && chosen.laneCount > 1 ? ` lane ${(chosen.laneIndex ?? 0) + 1}/${chosen.laneCount}` diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index e455c6b4f5..c17638bdfa 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -12,8 +12,10 @@ import { configuredWorkerLanesPerCredential, dispatchBee, drain, + endpointPoolProblems, finishDispatch, missingProviderRefusal, + POOL_KEY_STRIDE, prepareWorktree, recordDispatch, resolveWorkerProvider, @@ -31,6 +33,18 @@ const GENERIC_WORKER_KEYS = [ ), ] +// Additional endpoint pools (TRIOS_QUEEN_WORKER_POOL__*). Listed for the same +// reason as every other name here: one process runs every api test file, and a +// pool left behind makes the next file's "one endpoint" case read two. +const POOL_VARIABLES = [2, 3].flatMap((pool) => [ + ...['BASE_URL', 'MODEL', 'PROVIDER', 'CONTEXT', 'API_KEY'].map( + (name) => `TRIOS_QUEEN_WORKER_POOL_${pool}_${name}`, + ), + ...[2, 3, 4].map( + (index) => `TRIOS_QUEEN_WORKER_POOL_${pool}_API_KEY_${index}`, + ), +]) + const KEYS = [ 'ZAI_API_KEY', 'ZAI_API_KEY_2', @@ -50,7 +64,10 @@ const KEYS = [ 'TRIOS_QUEEN_WORKER_BASE_URL', 'TRIOS_QUEEN_WORKER_MODEL', 'TRIOS_QUEEN_WORKER_CONTEXT', + 'TRIOS_QUEEN_WORKER_LANES_PER_KEY', + 'TRIOS_QUEEN_MAX_WORKERS', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', + ...POOL_VARIABLES, ] afterEach(() => { @@ -240,6 +257,260 @@ describe('queen dispatch precheck', () => { }) }) + /** + * Measured on 2026-09-17: the deployment held ten working Z.ai keys and three + * working NVIDIA keys, and only the first ten could carry a bee, because one + * endpoint URL was the whole model and a key is only as good as the URL the + * bee sends it to. A numbered pool is a second endpoint with its own model + * and its own keys; the first pool is the unnumbered variables, unchanged. + */ + describe('additional endpoint pools', () => { + const ZAI_URL = 'https://api.z.ai/api/paas/v4' + const NVIDIA_URL = 'https://integrate.api.nvidia.com/v1' + const NVIDIA_MODEL = 'nvidia/nemotron-3-super-120b-a12b' + + const firstPool = (...keys: string[]) => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'zai' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = ZAI_URL + process.env.TRIOS_QUEEN_WORKER_MODEL = 'glm-4.5-flash' + process.env.TRIOS_QUEEN_WORKER_CONTEXT = '65536' + keys.forEach((value, index) => { + process.env[GENERIC_WORKER_KEYS[index]] = value + }) + } + const secondPool = (...keys: string[]) => { + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = `${NVIDIA_URL}/` + process.env.TRIOS_QUEEN_WORKER_POOL_2_MODEL = NVIDIA_MODEL + process.env.TRIOS_QUEEN_WORKER_POOL_2_CONTEXT = '131072' + keys.forEach((value, index) => { + const name = + index === 0 + ? 'TRIOS_QUEEN_WORKER_POOL_2_API_KEY' + : `TRIOS_QUEEN_WORKER_POOL_2_API_KEY_${index + 1}` + process.env[name] = value + }) + } + + it('sends each key to the endpoint of its own pool', () => { + firstPool('z1', 'z2', 'z3') + secondPool('n1', 'n2') + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 5, + lanesPerCredential: 1, + effectiveCapacity: 5, + }) + const taken: number[] = [] + const choices = Array.from({ length: 5 }, () => { + const choice = resolveWorkerProvider([...taken]) + taken.push(choice?.keyIndex ?? -1) + return choice + }) + // The first pool keeps the indices it has always had; pool 2 starts at + // the stride, so rows written before this change still mean the same key. + expect(choices.map((choice) => choice?.keyIndex)).toEqual([ + 0, + 1, + 2, + POOL_KEY_STRIDE, + POOL_KEY_STRIDE + 1, + ]) + expect(choices.map((choice) => choice?.apiKey)).toEqual([ + 'z1', + 'z2', + 'z3', + 'n1', + 'n2', + ]) + expect(choices[0]).toMatchObject({ + provider: 'zai', + baseUrl: ZAI_URL, + model: 'glm-4.5-flash', + contextWindow: 65536, + keyCount: 3, + poolNumber: 1, + poolCount: 2, + }) + // A Z.ai URL with an NVIDIA key is a 401 blamed on the work. Everything + // that travels to /chat comes from the SAME pool as the key. + expect(choices[3]).toMatchObject({ + provider: 'openai-compatible', + baseUrl: NVIDIA_URL, + model: NVIDIA_MODEL, + contextWindow: 131072, + keyCount: 2, + poolNumber: 2, + poolCount: 2, + }) + expect(resolveWorkerProvider(taken)?.exhausted).toBe(5) + }) + + it('rotates across pools after the last durable assignment', () => { + firstPool('z1', 'z2') + secondPool('n1', 'n2') + + expect(resolveWorkerProvider([], 1)?.keyIndex).toBe(POOL_KEY_STRIDE) + expect(resolveWorkerProvider([], POOL_KEY_STRIDE)?.keyIndex).toBe( + POOL_KEY_STRIDE + 1, + ) + expect(resolveWorkerProvider([], POOL_KEY_STRIDE + 1)?.keyIndex).toBe(0) + // A cursor naming a key that is no longer connected starts from the top + // instead of being read as a position in a list it was never part of. + expect(resolveWorkerProvider([], 2 * POOL_KEY_STRIDE + 7)?.keyIndex).toBe( + 0, + ) + }) + + it('gives every credential of every pool one bee before any key carries two', () => { + firstPool('z1', 'z2') + secondPool('n1') + process.env.TRIOS_QUEEN_WORKER_LANES_PER_KEY = '2' + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 3, + lanesPerCredential: 2, + effectiveCapacity: 6, + }) + const taken: number[] = [] + for (let bee = 0; bee < 6; bee++) { + const choice = resolveWorkerProvider( + [...taken], + taken[taken.length - 1], + ) + taken.push(choice?.keyIndex ?? -1) + } + expect(taken).toEqual([0, 1, POOL_KEY_STRIDE, 0, 1, POOL_KEY_STRIDE]) + expect(resolveWorkerProvider(taken)?.exhausted).toBe(6) + }) + + it('stays behind the policy ceiling however many pools are connected', () => { + firstPool('z1', 'z2', 'z3') + secondPool('n1', 'n2', 'n3') + + // TRIOS_QUEEN_MAX_WORKERS unset: the measured default of four. + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 6, + lanesPerCredential: 1, + effectiveCapacity: 4, + }) + expect(configuredWorkerCapacity()).toBe(4) + }) + + it('ignores a half-configured pool and says which variable is missing', () => { + firstPool('z1') + // A URL and keys but no model: the first pool's model name sent to + // another provider is a 404, so the pool is not connected at all. + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_2_API_KEY = 'n1' + // Keys with no endpoint to send them to. + process.env.TRIOS_QUEEN_WORKER_POOL_3_API_KEY = 'x1' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(resolveWorkerProvider([0])?.exhausted).toBe(1) + const problems = endpointPoolProblems() + expect(problems).toContain('TRIOS_QUEEN_WORKER_POOL_2_MODEL is not set') + expect(problems.join(' ')).toContain('TRIOS_QUEEN_WORKER_POOL_3_BASE_URL') + // Names only. A value in a diagnostic is a disclosure. + expect(problems.join(' ')).not.toContain('n1') + expect(problems.join(' ')).not.toContain('x1') + }) + + it('names an ignored pool when every connected key is busy', async () => { + firstPool('z1') + process.env.TRIOS_QUEEN_WORKER_POOL_2_BASE_URL = NVIDIA_URL + process.env.TRIOS_QUEEN_WORKER_POOL_2_API_KEY = 'n1' + const pool = { + query: async () => ({ rowCount: 1, rows: [] }), + } as unknown as Pool + + const outcome = await dispatchBee(pool, 1308, 'brief', [], [0]) + + expect(outcome.started).toBe(false) + expect(outcome.detail).toContain('TRIOS_QUEEN_WORKER_API_KEY_2') + expect(outcome.detail).toContain( + 'TRIOS_QUEEN_WORKER_POOL_2_MODEL is not set', + ) + expect(outcome.detail).not.toContain('n1') + }) + + it('refuses a pool whose provider is not a remote endpoint', () => { + firstPool('z1') + secondPool('n1') + process.env.TRIOS_QUEEN_WORKER_POOL_2_PROVIDER = 'ollama' + + expect(workerCapacityBreakdown().connectedCredentials).toBe(1) + expect(endpointPoolProblems().join(' ')).toContain( + 'TRIOS_QUEEN_WORKER_POOL_2_PROVIDER', + ) + }) + + it('counts one secret named by two pools as one credential', () => { + firstPool('shared', 'z2') + secondPool('shared', 'n2') + + expect(workerCapacityBreakdown().connectedCredentials).toBe(3) + const second = resolveWorkerProvider([0, 1]) + // The first pool that names a secret keeps it; pool 2 offers only 'n2'. + expect(second?.apiKey).toBe('n2') + expect(second?.keyIndex).toBe(POOL_KEY_STRIDE) + expect(second?.keyCount).toBe(1) + }) + + it('does not let a valid second pool take over from a first one with no key', () => { + firstPool() + secondPool('n1') + + // An explicit endpoint is authoritative, including its refusal. + expect(resolveWorkerProvider()).toBeNull() + expect(missingProviderRefusal()).toContain('TRIOS_QUEEN_WORKER_API_KEY') + }) + + it('keeps a local first endpoint as one measured lane and reads no pools', () => { + process.env.TRIOS_QUEEN_WORKER_PROVIDER = 'ollama' + process.env.TRIOS_QUEEN_WORKER_BASE_URL = 'http://ollama:11434/v1' + secondPool('n1', 'n2') + + expect(resolveWorkerProvider()?.provider).toBe('ollama') + expect(resolveWorkerProvider()?.poolCount).toBeUndefined() + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 1, + lanesPerCredential: 1, + effectiveCapacity: 1, + }) + }) + + it('leaves a single endpoint exactly as it was', () => { + firstPool('z1', 'z2') + + const choice = resolveWorkerProvider([0]) + expect(choice?.keyIndex).toBe(1) + expect(choice?.poolNumber).toBeUndefined() + expect(choice?.poolCount).toBeUndefined() + expect(endpointPoolProblems()).toEqual([]) + }) + + it('keeps the breakdown closed when several pools are connected', () => { + firstPool('planted-secret-z') + secondPool('planted-secret-n') + const breakdown = workerCapacityBreakdown() as unknown as Record< + string, + unknown + > + + expect(Object.keys(breakdown).sort()).toEqual([ + 'connectedCredentials', + 'effectiveCapacity', + 'lanesPerCredential', + ]) + const serialized = JSON.stringify(breakdown) + expect(serialized).not.toContain('planted-secret') + expect(serialized).not.toContain('POOL_2') + expect(serialized).not.toContain('nvidia') + }) + }) + // Four bees on one key share one rate limit, so the swarm's real ceiling // becomes whatever that key allows rather than what the Queen permits - and // the 429 arrives blamed on the work. From 78936ad4787c91768a0364ed64fc4e5ea708ed5a Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 17 Sep 2026 17:31:24 +0700 Subject: [PATCH 2/3] fix(queen): the seventeenth key was read by nothing, because 16 was where a loop ended keysFor() read the unsuffixed variable and then _2 ... _16. Sixteen was never a measurement: it is where the loop happened to stop, and it equals the policy ceiling on bees only by coincidence. They are different quantities. The ceiling bounds how many bees RUN. The key list bounds how many credentials the rotation may SPREAD them over - and a free key refuses its third concurrent request in under half a second (1302), so a swarm at the ceiling wants more keys than bees, not the same number. A TRIOS_QUEEN_WORKER_API_KEY_17 was therefore read by nothing and reported by nothing: it looks configured in a variable editor and supplies nothing, the zero-length-key trap one step to the right. The suffix now runs to MAX_KEYS_PER_POOL = 1000, for every pool and for the legacy provider variables alike (they share keysFor). Widening the list does not widen the swarm: queenWorkerLimit() still answers at most 16, and a test pins seventeen credentials to a capacity of sixteen. POOL_KEY_STRIDE moves from 1000 to 10000 with it. The durable index of pool n starts at (n - 1) * stride, so a pool that could hold `stride` keys would have its last key collide with the next pool's first. Nothing has been written with the old stride - it exists only in this pull request - and a test now holds MAX_KEYS_PER_POOL below POOL_KEY_STRIDE so the two cannot drift. Dispatch tests: 81 pass (78 before this commit). Typecheck clean. Co-Authored-By: Claude Fable 5.1 --- .../server/src/api/services/queen-dispatch.ts | 26 ++++++++--- .../server/tests/api/queen-dispatch.test.ts | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 95028b17e7..66c69eea92 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -231,7 +231,20 @@ export interface WorkerProvider { * trimmed value is also the one stored: a key that authenticates never needed * its padding, and handing the trimmed form out keeps the count and the * selection - which both read this list - from ever disagreeing. + * + * The suffix runs to MAX_KEYS_PER_POOL. It used to stop at 16, and 16 was never + * a measurement: it was where the loop happened to end, equal by coincidence to + * the policy ceiling on bees. They are different quantities. The ceiling bounds + * how many bees RUN; this bounds how many credentials the rotation may SPREAD + * them over, and a free key refuses its third concurrent request in under half + * a second (`1302`), so a swarm at the ceiling wants more keys than bees, not + * the same number. A seventeenth variable used to be read by nothing and + * reported by nothing, which is the zero-length-key trap again: it looks + * configured and supplies nothing. Widening the list does not widen the swarm - + * `queenWorkerLimit` still decides that. */ +export const MAX_KEYS_PER_POOL = 1000 + function keysFor(envVar: string): string[] { const keys: string[] = [] const seen = new Set() @@ -242,7 +255,7 @@ function keysFor(envVar: string): string[] { keys.push(trimmed) } admit(process.env[envVar]) - for (let i = 2; i <= 16; i++) { + for (let i = 2; i <= MAX_KEYS_PER_POOL; i++) { admit(process.env[`${envVar}_${i}`]) } return keys @@ -485,7 +498,7 @@ function availableKeyIndex( * TRIOS_QUEEN_WORKER_POOL__MODEL the model THAT endpoint serves (required) * TRIOS_QUEEN_WORKER_POOL__PROVIDER openai-compatible (default) or zai * TRIOS_QUEEN_WORKER_POOL__CONTEXT that model's context window - * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... _API_KEY_16 + * TRIOS_QUEEN_WORKER_POOL__API_KEY, _API_KEY_2 ... (MAX_KEYS_PER_POOL) * * A pool is REMOTE credentials by definition. A local Ollama stays what it was * measured to be - one inference slot - and does not join a credential pool, @@ -508,12 +521,13 @@ const REMOTE_ENDPOINT_PROVIDERS = new Set(['openai-compatible', 'zai']) * which credentials are busy. Keys of the first pool keep the index they have * always had (0, 1, 2 ...), so rows written before this change still mean what * they meant. A key of pool n is `(n - 1) * POOL_KEY_STRIDE + position`: pool - * 2's first key is 1000. The pool NUMBER is used, not its rank among the pools + * 2's first key is 10000. The pool NUMBER is used, not its rank among the pools * that happen to be valid today, so disconnecting pool 2 does not silently - * rename every busy key of pool 3. A pool holds at most 16 keys, so the stride - * can never be reached from inside one. + * rename every busy key of pool 3. A pool holds at most MAX_KEYS_PER_POOL keys, + * which is below the stride, so the stride can never be reached from inside + * one - a test holds the two constants to that. */ -export const POOL_KEY_STRIDE = 1000 +export const POOL_KEY_STRIDE = 10_000 interface EndpointPool { /** 1-based; 1 is the pool made of the unnumbered variables. */ diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index c17638bdfa..7b0d1412a6 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -14,6 +14,7 @@ import { drain, endpointPoolProblems, finishDispatch, + MAX_KEYS_PER_POOL, missingProviderRefusal, POOL_KEY_STRIDE, prepareWorktree, @@ -66,6 +67,10 @@ const KEYS = [ 'TRIOS_QUEEN_WORKER_CONTEXT', 'TRIOS_QUEEN_WORKER_LANES_PER_KEY', 'TRIOS_QUEEN_MAX_WORKERS', + // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. + 'TRIOS_QUEEN_WORKER_API_KEY_17', + 'TRIOS_QUEEN_WORKER_API_KEY_1000', + 'TRIOS_QUEEN_WORKER_API_KEY_1001', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', ...POOL_VARIABLES, ] @@ -481,6 +486,45 @@ describe('queen dispatch precheck', () => { }) }) + it('reads a key past the sixteenth, up to the bound and not beyond it', () => { + firstPool('z1') + // 16 was where the loop ended, not a measurement: a seventeenth variable + // was read by nothing and reported by nothing. + process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1000 = 'z1000' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1001 = 'past-the-bound' + process.env.TRIOS_QUEEN_MAX_WORKERS = '16' + + expect(MAX_KEYS_PER_POOL).toBe(1000) + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 3, + lanesPerCredential: 1, + effectiveCapacity: 3, + }) + expect(resolveWorkerProvider([0])?.apiKey).toBe('z17') + expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1000') + expect(resolveWorkerProvider([0, 1, 2])?.exhausted).toBe(3) + }) + + it('widens the key list without widening the swarm', () => { + firstPool(...Array.from({ length: 16 }, (_, index) => `z${index + 1}`)) + process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' + process.env.TRIOS_QUEEN_MAX_WORKERS = '64' + + // Seventeen credentials, and the policy ceiling still answers sixteen. + expect(workerCapacityBreakdown()).toEqual({ + connectedCredentials: 17, + lanesPerCredential: 1, + effectiveCapacity: 16, + }) + }) + + it('keeps a whole pool below the stride that separates pools', () => { + // The durable index of pool n starts at (n - 1) * POOL_KEY_STRIDE. If a + // pool could hold that many keys, its last key would BE pool n + 1's first. + expect(MAX_KEYS_PER_POOL).toBeLessThan(POOL_KEY_STRIDE) + }) + it('leaves a single endpoint exactly as it was', () => { firstPool('z1', 'z2') From 2ac9c110df25b61f4f6cfbfa1af31cca20a26307 Mon Sep 17 00:00:00 2001 From: Dmitrii Fedorov Date: Thu, 17 Sep 2026 17:42:01 +0700 Subject: [PATCH 3/3] fix(queen): a key bound of a thousand could sit below a worker ceiling of 1024 MAX_KEYS_PER_POOL becomes 1024. With one lane per credential a swarm of N bees needs N keys, so a key bound under the worker bound is a second, hidden ceiling that reports nothing when it is hit. The operator is asking for the policy ceiling to be raised to 1024; a round thousand would have been that hidden ceiling the day it was. The worker ceiling itself is NOT changed here - it is a policy with a Swift mirror, and raising it is a separate, deliberate decision. Dispatch tests: 81 pass. Typecheck clean. Co-Authored-By: Claude Fable 5.1 --- .../apps/server/src/api/services/queen-dispatch.ts | 7 ++++++- .../apps/server/tests/api/queen-dispatch.test.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts index 66c69eea92..d934860bf8 100644 --- a/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts +++ b/trios/agent-server/apps/server/src/api/services/queen-dispatch.ts @@ -242,8 +242,13 @@ export interface WorkerProvider { * reported by nothing, which is the zero-length-key trap again: it looks * configured and supplies nothing. Widening the list does not widen the swarm - * `queenWorkerLimit` still decides that. + * + * 1024 rather than a round thousand so that this bound can never sit BELOW a + * worker ceiling expressed as a power of two: with one lane per credential a + * swarm of N bees needs N keys, and a key bound under the worker bound would be + * a second, hidden ceiling that reports nothing when it is hit. */ -export const MAX_KEYS_PER_POOL = 1000 +export const MAX_KEYS_PER_POOL = 1024 function keysFor(envVar: string): string[] { const keys: string[] = [] diff --git a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts index 7b0d1412a6..bae5d90f94 100644 --- a/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts +++ b/trios/agent-server/apps/server/tests/api/queen-dispatch.test.ts @@ -69,8 +69,8 @@ const KEYS = [ 'TRIOS_QUEEN_MAX_WORKERS', // The far end of the key list (MAX_KEYS_PER_POOL) and the first name past it. 'TRIOS_QUEEN_WORKER_API_KEY_17', - 'TRIOS_QUEEN_WORKER_API_KEY_1000', - 'TRIOS_QUEEN_WORKER_API_KEY_1001', + 'TRIOS_QUEEN_WORKER_API_KEY_1024', + 'TRIOS_QUEEN_WORKER_API_KEY_1025', 'TRIOS_ZAI_CONCURRENCY_PER_KEY', ...POOL_VARIABLES, ] @@ -491,18 +491,18 @@ describe('queen dispatch precheck', () => { // 16 was where the loop ended, not a measurement: a seventeenth variable // was read by nothing and reported by nothing. process.env.TRIOS_QUEEN_WORKER_API_KEY_17 = 'z17' - process.env.TRIOS_QUEEN_WORKER_API_KEY_1000 = 'z1000' - process.env.TRIOS_QUEEN_WORKER_API_KEY_1001 = 'past-the-bound' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1024 = 'z1024' + process.env.TRIOS_QUEEN_WORKER_API_KEY_1025 = 'past-the-bound' process.env.TRIOS_QUEEN_MAX_WORKERS = '16' - expect(MAX_KEYS_PER_POOL).toBe(1000) + expect(MAX_KEYS_PER_POOL).toBe(1024) expect(workerCapacityBreakdown()).toEqual({ connectedCredentials: 3, lanesPerCredential: 1, effectiveCapacity: 3, }) expect(resolveWorkerProvider([0])?.apiKey).toBe('z17') - expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1000') + expect(resolveWorkerProvider([0, 1])?.apiKey).toBe('z1024') expect(resolveWorkerProvider([0, 1, 2])?.exhausted).toBe(3) })