From 92040e8dd8c35bf14346dc9303166020fedc262b Mon Sep 17 00:00:00 2001 From: Vadym O Date: Thu, 15 Oct 2026 20:47:34 +0800 Subject: [PATCH 1/3] fix(combos): a spent generic OAuth account cools the account, not the combo target The per-request account budget can end a generic OAuth pool's rotation before it reaches every stored account, so a 429 reaches the combo layer while a sibling is still healthy. The combo then cooled the whole target for that account's Retry-After (up to the 24h server-delay ceiling) and sent traffic to the fallback target, or answered 503. Resolve the refused account from the child's accountLogLabel and hand it to the pool's own rotator, which cools it under the pool's Retry-After policy. The target is spared only when the rotator names an eligible sibling; a lone account or a fully cooled pool still cools the target. Anthropic OAuth and Codex attribution are unchanged. API-key pools need no change: their 429 rotation walks every key inside the attempt. Co-Authored-By: Claude Code --- scripts/test-layout/layout.json | 1 + src/server/responses/core-combo.ts | 29 +++- structure/runtime.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- ...neric-oauth-combo-account-cooldown.test.ts | 131 ++++++++++++++++++ tests/oauth/generic-oauth-failover.test.ts | 9 +- 6 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 tests/oauth/generic-oauth-combo-account-cooldown.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 9925e111815..b00be853cf9 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -938,6 +938,7 @@ "gemini-web-search.test.ts": "adapters/google", "generic-session-affinity.test.ts": "oauth", "server-generic-oauth-affinity.test.ts": "server", + "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot", "github-copilot-account-origin.test.ts": "providers/github-copilot", diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index f4714f0c9b7..2ac6b4be012 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -52,14 +52,15 @@ import { import { hasUnreadableEncryptedAgentTask } from "./encrypted-payload"; import { routeConcreteModel, comboRouteDecisionTrace } from "../../router"; import { memoryModelRouteReason } from "./memory-models"; -import { poolAccountProviderLabel } from "../../providers/label"; +import { baseProviderLabel, poolAccountProviderLabel } from "../../providers/label"; import { getAccountSet } from "../../oauth/store"; import { configuredAnthropicInstance } from "../../providers/anthropic-instance"; import { anthropicRoutingFor, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; -import { codexAccountLogLabel } from "../../codex/account-label"; +import { codexAccountLogLabel, oauthAccountLogLabel } from "../../codex/account-label"; +import { isGenericFailoverProvider, rotateGenericOAuthAccountOn429 } from "../../oauth/generic-account-failover"; import { codexQuotaScopeForModel, getCodexQuotaHealthSnapshot } from "../../codex/routing"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery"; @@ -135,6 +136,26 @@ function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId ? label : undefined; } +/** + * The generic OAuth pool's counterpart of `cooledPoolAccountLabel`. Its 429 names one account + * through `accountLogLabel`, but the per-request account budget can end the pool's own rotation + * before the last refused account is recorded. Hand that account to the pool's rotator, which + * cools it under the pool's own Retry-After policy, and spare the target only when the pool still + * has an eligible sibling. A lone account or an all-cooled pool returns null and cools the target. + */ +function cooledGenericOAuthAccountLabel( + config: OcxConfig, providerName: string, modelId: string, label: string | undefined, retryAfter: string | undefined, +): string | undefined { + const provider = config.providers[providerName]; + if (!label || !provider || !isGenericFailoverProvider(providerName, provider)) return undefined; + const base = baseProviderLabel(providerName); + const matches = getAccountSet(providerName)?.accounts.filter(account => + oauthAccountLogLabel(account.id, base) === label) ?? []; + return matches.length === 1 + && rotateGenericOAuthAccountOn429(config, providerName, matches[0]!.id, retryAfter, Date.now(), modelId) + ? label : undefined; +} + /** * True only for the Anthropic OAuth pool's OWN "all accounts cooled" 429. Its Retry-After is the @@ -1140,7 +1161,9 @@ export async function executeComboResponses( pick.target.provider, pick.target.model, poolAccountProviderLabel(childLog.provider, pick.target.provider), - ); + ) ?? (failure.response.status === 429 && failureCooldownScope === "target" + ? cooledGenericOAuthAccountLabel(config, pick.target.provider, pick.target.model, childLog.accountLogLabel, failure.retryAfter) + : undefined); // The target still cools on the pool's own refusal (the anti-hammer guard), but for the local // fallback rather than the pool's restated per-account Retry-After. const poolLocalRefusal = isAnthropicPoolLocalRefusal( diff --git a/structure/runtime.md b/structure/runtime.md index dc720c31f82..c281555d49d 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. ## Combo default effort precedence diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f5047535934..432cc842a7a 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -661,7 +661,7 @@ "gemini-inline.test.ts": "images", "gemini-web-search.test.ts": "adapters/google", "generic-session-affinity.test.ts": "oauth", "server-generic-oauth-affinity.test.ts": "server", - "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", + "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot", diff --git a/tests/oauth/generic-oauth-combo-account-cooldown.test.ts b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts new file mode 100644 index 00000000000..0ec326ed748 --- /dev/null +++ b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts @@ -0,0 +1,131 @@ +/** + * A generic OAuth pool's spent account must not park the whole combo target. + * + * The per-request account budget stops the pool's own rotation before it reaches every stored + * account, so a 429 can reach the combo layer while a sibling is still healthy. Cooling the target + * for that account's `Retry-After` (up to the 24h server-delay ceiling) would answer zero-attempt + * 503s, or pay for a fallback target, while the pool still has an account that answers. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearComboSelectionState, clearComboTargetCooldowns, isComboTargetInCooldown } from "../../src/combos"; +import { clearGenericFailoverHealth, eligibleFailoverAccounts } from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { clearAccountQuotaCache } from "../../src/providers/quota"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { chatSuccess } from "../helpers/combo-failover-upstream"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const TARGET = { provider: "xai", model: "m1" }; +const originalHome = process.env.OPENCODEX_HOME; +const originalFetch = globalThis.fetch; +let home = ""; +let releaseSpendHome: (() => void) | undefined; +/** Account index per upstream send (`B` for the fallback target). */ +let sends: string[] = []; + +async function seed(count: number): Promise { + for (let index = 1; index <= count; index++) { + await saveCredential("xai", { + access: `synthetic-xai-access-${index}`, refresh: `synthetic-xai-refresh-${index}`, + expires: Date.now() + 3_600_000, accountId: `synthetic-xai-${index}`, + } as never, { addAccount: true }); + } + await setActiveAccount("xai", getAccountSet("xai")!.accounts[0]!.id); +} + +/** Accounts at or below `spentThrough` answer a one-hour 429; the rest answer 200. */ +function upstream(spentThrough: number): void { + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input instanceof Request ? input.url : input); + if (url.startsWith("https://fallback.test/")) { + sends.push("B"); + return chatSuccess("fallback answer", "m2"); + } + const index = Number(new Headers(init?.headers).get("authorization")?.split("-").pop()); + sends.push(String(index)); + return index <= spentThrough + ? Response.json({ error: { message: "synthetic rate limit" } }, { status: 429, headers: { "retry-after": "3600" } }) + : chatSuccess("pool answer", "m1"); + }) as typeof fetch; +} + +const config: OcxConfig = { + port: 0, defaultProvider: "xai", + providers: { + xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "oauth", models: ["m1"] }, + fallback: { adapter: "openai-chat", baseUrl: "https://fallback.test/v1", authMode: "key", apiKey: "synthetic-fallback-key", models: ["m2"] }, + }, + combos: { pooled: { strategy: "failover", targets: [TARGET, { provider: "fallback", model: "m2" }] } }, +}; + +async function post(): Promise { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/pooled", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }); + expect(response.status).toBe(200); + return response.text(); +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-generic-combo-cooldown-")); + process.env.OPENCODEX_HOME = home; + sends = []; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + clearResponseStateForTests(); + releaseSpendHome = acquireOwnedSpendHome(); +}); + +afterEach(async () => { + await flushResponseState(); + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + clearResponseStateForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + if (home) removeTreeWithRetry(home); +}); + +test("a spent generic OAuth account cools the account, not the combo target", async () => { + await seed(5); + upstream(4); + + expect(await post()).toContain("fallback answer"); + // Precondition: the request budget ended the pool's rotation before the healthy fifth account. + expect(sends).not.toContain("5"); + expect(sends.at(-1)).toBe("B"); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + // Every account this request reached is cooled by the pool, the last one included. + expect(eligibleFailoverAccounts("xai")).toEqual([getAccountSet("xai")!.accounts[4]!.id]); + + // The pool, not the fallback target, serves the next request. Without proactive preference the + // pool may still open on its cooled active account once before rotating; that is the pool's own + // contract and stays bounded by its rotation. + sends = []; + expect(await post()).toContain("pool answer"); + expect(sends).not.toContain("B"); + expect(sends.at(-1)).toBe("5"); +}, 20_000); + +test("every generic OAuth account spent still cools the combo target", async () => { + await seed(2); + upstream(2); + + expect(await post()).toContain("fallback answer"); + expect(sends).toEqual(["1", "2", "B"]); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); +}, 20_000); diff --git a/tests/oauth/generic-oauth-failover.test.ts b/tests/oauth/generic-oauth-failover.test.ts index bb4150200f5..63a253f5c60 100644 --- a/tests/oauth/generic-oauth-failover.test.ts +++ b/tests/oauth/generic-oauth-failover.test.ts @@ -559,8 +559,11 @@ describe("sidecar on429 wiring", () => { // The counts differ by rotator because the recovery sites differ, and each number is a // statement about which providers can recover where: // - // generic = 5: streaming loop, continuation loop, sidecar hook, runTurn preflight, - // native Responses passthrough. The Antigravity 403 verify-account + // generic = 6: streaming loop, continuation loop, sidecar hook, runTurn preflight, + // native Responses passthrough, plus the combo failure attribution in + // core-combo.ts, which sends nothing: it records the refused account the + // request budget left uncooled and spares the combo target only when the + // rotator names a sibling. The Antigravity 403 verify-account // arm deliberately does NOT use this rotator: a verification refusal // must not record rate-limit cooldown semantics, so it moves via // rotateAntigravityAccountOnAuthRefusal instead. The new default @@ -581,7 +584,7 @@ describe("sidecar on429 wiring", () => { // // Adding a recovery site means deciding, deliberately, which rotators it needs and // updating the matching number. That decision is the thing this test exists to force. - expect(counts.generic).toBe(5); + expect(counts.generic).toBe(6); expect(counts.anthropic).toBe(3); expect(counts.key).toBe(7); }); From 0dadd7ffd9ab76e8076386a2118f3ae53f9ab228 Mon Sep 17 00:00:00 2001 From: Vadym O Date: Thu, 15 Oct 2026 23:16:37 +0800 Subject: [PATCH 2/3] fix(combos): one Anthropic account's 402 credit refusal cools the account, not the combo target The Anthropic OAuth pool rotates only on 401/403/429, so a 402 carrying the credit-balance refusal reaches the combo with the refused account still selectable. The combo classifies 402 as provider-scoped and cooled every anthropic target in the combo, sending traffic to the fallback (or a zero-attempt 503) while a sibling account still answered. When the failed child identifies one pool account and the failure carries the exact credit-balance wording, record that account's refusal through the pool's own recordAnthropicAccountRefusal (the entitlement backoff a 403 credit refusal already gets) and skip the target cooldown only while the pool still has an eligible account. A lone account, an exhausted pool, an unidentified account and every other 402 keep the provider-scoped cooldown. Co-Authored-By: Claude Code --- src/combos/resolve.ts | 7 +-- src/server/responses/core-combo.ts | 35 +++++++++++--- structure/runtime.md | 2 +- .../anthropic-combo-account-cooldown.test.ts | 47 +++++++++++++++++++ 4 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 59559154706..c23d6b46a6c 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -408,9 +408,10 @@ export function advanceComboAfterFailure( // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is // holding the target back, the target cooldown still does. - const accountScoped = options.cooldownScope === "target" - && options.status === 429 - && options.failedAccount !== undefined; + // A 402 credit refusal is the provider-scoped twin: the caller names an account only after the + // pool has cooled it and still holds an eligible sibling. + const accountScoped = options.failedAccount !== undefined + && (options.status === 429 ? options.cooldownScope === "target" : options.status === 402); // "none" records no cooldown at all: the failure described the request, not the target, so // the target must stay immediately selectable for the next (differently shaped) request. if (options.cooldownScope !== "none" && !accountScoped) { diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 2ac6b4be012..0e01f8c4443 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -55,6 +55,7 @@ import { memoryModelRouteReason } from "./memory-models"; import { baseProviderLabel, poolAccountProviderLabel } from "../../providers/label"; import { getAccountSet } from "../../oauth/store"; import { configuredAnthropicInstance } from "../../providers/anthropic-instance"; +import { isAnthropicBillingFailure } from "../../providers/anthropic-billing-refusal"; import { anthropicRoutingFor, formatAnthropicProviderForLog, @@ -156,6 +157,27 @@ function cooledGenericOAuthAccountLabel( ? label : undefined; } +/** + * A 402 credit-balance refusal names one Anthropic OAuth account -- the evidence the pool already + * accepts on a 403 -- but the pool rotates only on 401/403/429, so the 402 reaches the combo with + * that account still selectable. Cool it under the pool's entitlement backoff and spare the target + * only while a sibling is eligible; a lone account or an exhausted pool still cools the target. + */ +function creditRefusedAnthropicAccountLabel( + config: OcxConfig, providerName: string, modelId: string, label: string | undefined, + message: string, code: string | null | undefined, +): string | undefined { + const instance = configuredAnthropicInstance(config, providerName); + if (!label || !instance || config.providers[providerName]?.authMode !== "oauth" + || !isAnthropicBillingFailure(402, message, code)) return undefined; + const routing = anthropicRoutingFor(instance); + const matches = getAccountSet(instance)?.accounts.filter(account => + formatAnthropicProviderForLog(instance, account.id) === label) ?? []; + return matches.length === 1 + && routing.recordAnthropicAccountRefusal(config, matches[0]!.id, 403, null) + && routing.getEligibleAnthropicAccounts(Date.now(), modelId).length > 0 + ? label : undefined; +} /** * True only for the Anthropic OAuth pool's OWN "all accounts cooled" 429. Its Retry-After is the @@ -1156,12 +1178,13 @@ export async function executeComboResponses( const failedTargetKey = targetKey(pick.target); let failedTargetCooldownRecorded = false; // The dispatch rewrote this to name the pool account that actually served the turn. - const failedAccount = cooledPoolAccountLabel( - config, - pick.target.provider, - pick.target.model, - poolAccountProviderLabel(childLog.provider, pick.target.provider), - ) ?? (failure.response.status === 429 && failureCooldownScope === "target" + const poolAccountLabel = poolAccountProviderLabel(childLog.provider, pick.target.provider); + const failedAccount = failure.response.status === 402 + ? creditRefusedAnthropicAccountLabel( + config, pick.target.provider, pick.target.model, poolAccountLabel, failure.classificationText, failure.upstreamCode, + ) + : cooledPoolAccountLabel(config, pick.target.provider, pick.target.model, poolAccountLabel) + ?? (failure.response.status === 429 && failureCooldownScope === "target" ? cooledGenericOAuthAccountLabel(config, pick.target.provider, pick.target.model, childLog.accountLogLabel, failure.retryAfter) : undefined); // The target still cools on the pool's own refusal (the anti-hammer guard), but for the local diff --git a/structure/runtime.md b/structure/runtime.md index c281555d49d..2f00e42e2d7 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. ## Combo default effort precedence diff --git a/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts index 79d589407c4..087a88017d8 100644 --- a/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts +++ b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts @@ -182,3 +182,50 @@ test("a lone OAuth account without pool cooldown still cools the combo target", expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); expect(sentTokens).toHaveLength(1); }); + +/** The pool's own credit-balance wording (`isAccountRefusal`), on a 402. */ +function creditRefused(): Response { + return Response.json({ type: "error", error: { + type: "billing_error", + message: "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", + } }, { status: 402 }); +} + +test("one account's 402 credit refusal does not cool the combo target while a sibling is eligible", async () => { + const ids = await seed(2); + const config = configFor(() => answered()); + const anthropic = config.providers.anthropic as OcxProviderConfig & { fetch: typeof fetch }; + // Only the first account is out of credit; the second still answers. + anthropic.fetch = (async (_input, init) => { + const token = String(new Headers(init?.headers).get("authorization")); + sentTokens.push(token); + return token === `Bearer ${credential(0).access}` ? creditRefused() : answered(); + }) as typeof fetch; + const fallbackSends: string[] = []; + config.providers.fallback = { + adapter: "anthropic", baseUrl: "https://fallback.test", authMode: "key", apiKey: "synthetic-fallback-key", + models: [MODEL], fetch: (async () => { fallbackSends.push("B"); return answered(); }) as typeof fetch, + } as OcxProviderConfig; + config.combos = { pooled: { strategy: "failover", targets: [TARGET, { provider: "fallback", model: MODEL }] } }; + + expect((await post(config)).status).toBe(200); + expect(sentTokens).toEqual([`Bearer ${credential(0).access}`]); + expect(fallbackSends).toEqual(["B"]); + // The refusal names one account: the pool cools it, and the target stays usable for the other. + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + expect(getAnthropicAccountHealthSnapshot(ids[0]!)).not.toBeNull(); + + sentTokens = []; + expect((await post(config)).status).toBe(200); + expect(sentTokens).toEqual([`Bearer ${credential(1).access}`]); + expect(fallbackSends).toEqual(["B"]); +}, 20_000); + +test("402 credit refusals from every pool account still cool the combo target", async () => { + await seed(2); + const config = configFor(creditRefused); + // The first refusal leaves a sibling and spares the target; the second leaves none. + for (let attempt = 0; attempt < 2; attempt++) expect((await post(config)).status).toBe(402); + expect(sentTokens).toHaveLength(2); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); +}, 20_000); From 1cd8438f60fcbbcfbc2301c2ef3e8d421b185b5f Mon Sep 17 00:00:00 2001 From: Vadym O Date: Fri, 16 Oct 2026 11:36:12 +0800 Subject: [PATCH 3/3] fix(combos): peek the round-robin ring in the combo's sibling check The combo layer asks the generic OAuth rotator only whether a healthy sibling exists; it never dispatches to the account the rotator names. Under pool.kernel round-robin the rotator committed that pick, advancing the ring and making it sticky to an account no request was sent to. Add a `commit` flag to rotateGenericOAuthAccountOn429 (default true). The combo passes false: the failed account is still cooled, but the ring is only peeked. Co-Authored-By: Claude Code --- src/oauth/generic-account-failover.ts | 11 +++++++---- src/server/responses/core-combo.ts | 5 +++-- ...eneric-oauth-combo-account-cooldown.test.ts | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index c8cbd8e042c..9fca39601c3 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -483,7 +483,8 @@ export function noteGenericPoolSelection( * Cool the account that actually 429'd and name the next eligible one, or null. * * Returns the id only; the caller mints the credential so a failed refresh does not leave the - * cooldown applied to an account we then could not use. + * cooldown applied to an account we then could not use. `commit: false` still cools the failed + * account but only peeks the round-robin ring, for callers that will not dispatch the answer. */ export function rotateGenericOAuthAccountOn429( config: OcxConfig, @@ -492,9 +493,10 @@ export function rotateGenericOAuthAccountOn429( retryAfterHeader: string | null | undefined, now = Date.now(), requestedModelId?: string | null, + commit = true, ): string | null { return rotateGenericOAuthAccountOnRefusal(config, providerName, failedAccountId, "rate", - retryAfterHeader, now, requestedModelId); + retryAfterHeader, now, requestedModelId, undefined, commit); } /** Kiro adds account-scoped refusal classes; all other providers retain their 429 policy. */ @@ -507,6 +509,7 @@ export function rotateGenericOAuthAccountOnRefusal( now = Date.now(), requestedModelId?: string | null, monthlyCooldownMs?: number, + commit = true, ): string | null { if (kind === "other" || (providerName !== "kiro" && kind !== "rate")) return null; if (!isGenericOAuthFailoverEnabled(config, providerName)) return null; @@ -566,8 +569,8 @@ export function rotateGenericOAuthAccountOnRefusal( } if (strategy === "round-robin") { // PICK here, not peek: the failure already happened and this answer is the one being used, - // so the ring genuinely advances. - return pickRoundRobinAccount( + // so the ring genuinely advances. A caller that only asks whether a sibling exists peeks. + return (commit ? pickRoundRobinAccount : peekRoundRobinAccount)( genericPoolKey(providerName), candidates, genericStickyLimit(config, providerName), diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index 0e01f8c4443..6a1b2c84fb7 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -142,7 +142,8 @@ function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId * through `accountLogLabel`, but the per-request account budget can end the pool's own rotation * before the last refused account is recorded. Hand that account to the pool's rotator, which * cools it under the pool's own Retry-After policy, and spare the target only when the pool still - * has an eligible sibling. A lone account or an all-cooled pool returns null and cools the target. + * has an eligible sibling. The rotator only peeks round-robin here: nothing is sent to that sibling. + * A lone account or an all-cooled pool returns null and cools the target. */ function cooledGenericOAuthAccountLabel( config: OcxConfig, providerName: string, modelId: string, label: string | undefined, retryAfter: string | undefined, @@ -153,7 +154,7 @@ function cooledGenericOAuthAccountLabel( const matches = getAccountSet(providerName)?.accounts.filter(account => oauthAccountLogLabel(account.id, base) === label) ?? []; return matches.length === 1 - && rotateGenericOAuthAccountOn429(config, providerName, matches[0]!.id, retryAfter, Date.now(), modelId) + && rotateGenericOAuthAccountOn429(config, providerName, matches[0]!.id, retryAfter, Date.now(), modelId, false) ? label : undefined; } diff --git a/tests/oauth/generic-oauth-combo-account-cooldown.test.ts b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts index 0ec326ed748..b8ec163c75d 100644 --- a/tests/oauth/generic-oauth-combo-account-cooldown.test.ts +++ b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts @@ -12,6 +12,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { clearComboSelectionState, clearComboTargetCooldowns, isComboTargetInCooldown } from "../../src/combos"; import { clearGenericFailoverHealth, eligibleFailoverAccounts } from "../../src/oauth/generic-account-failover"; +import { genericPoolKey, peekRoundRobinAccount } from "../../src/oauth/pool-kernel"; import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; import { clearAccountQuotaCache } from "../../src/providers/quota"; import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; @@ -129,3 +130,20 @@ test("every generic OAuth account spent still cools the combo target", async () expect(sends).toEqual(["1", "2", "B"]); expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); }, 20_000); + +test("the combo's sibling check does not advance a round-robin pool onto an unsent account", async () => { + await seed(5); + upstream(4); + const roundRobin: OcxConfig = { ...config, pool: { kernel: true }, providers: { ...config.providers, + xai: { ...config.providers.xai!, oauthAccountFailover: { strategy: "round-robin", stickyLimit: 3 } } } }; + const ids = getAccountSet("xai")!.accounts.map(account => account.id); + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/pooled", input: "Answer briefly", stream: false }), + }), roundRobin, { model: "", provider: "" }); + expect(await response.text()).toContain("fallback answer"); + expect(sends).not.toContain("5"); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + // Account 5 was only probed for eligibility, never sent; the ring must not stick to it. + expect(peekRoundRobinAccount(genericPoolKey("xai"), ids, 3)).not.toBe(ids[4]); +}, 20_000);