fix(combos): one Anthropic account's 402 credit refusal cools the account, not the combo target

scripts/test-layout/layout.json

@@ -938,6 +938,7 @@
     "gemini-web-search.test.ts": "adapters/google",
     "generic-session-affinity.test.ts": "oauth",
     "server-generic-oauth-affinity.test.ts": "server",
+    "generic-oauth-combo-account-cooldown.test.ts": "oauth",
     "generic-oauth-failover.test.ts": "oauth",
     "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot",
     "github-copilot-account-origin.test.ts": "providers/github-copilot",

src/combos/resolve.ts

@@ -408,9 +408,10 @@ export function advanceComboAfterFailure(
   // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks
   // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is
   // holding the target back, the target cooldown still does.
-  const accountScoped = options.cooldownScope === "target"
-    && options.status === 429
-    && options.failedAccount !== undefined;
+  // A 402 credit refusal is the provider-scoped twin: the caller names an account only after the
+  // pool has cooled it and still holds an eligible sibling.
+  const accountScoped = options.failedAccount !== undefined
+    && (options.status === 429 ? options.cooldownScope === "target" : options.status === 402);
   // "none" records no cooldown at all: the failure described the request, not the target, so
   // the target must stay immediately selectable for the next (differently shaped) request.
   if (options.cooldownScope !== "none" && !accountScoped) {

src/oauth/generic-account-failover.ts

@@ -483,7 +483,8 @@ export function noteGenericPoolSelection(
  * Cool the account that actually 429'd and name the next eligible one, or null.
  *
  * Returns the id only; the caller mints the credential so a failed refresh does not leave the
- * cooldown applied to an account we then could not use.
+ * cooldown applied to an account we then could not use. `commit: false` still cools the failed
+ * account but only peeks the round-robin ring, for callers that will not dispatch the answer.
  */
 export function rotateGenericOAuthAccountOn429(
   config: OcxConfig,
@@ -492,9 +493,10 @@ export function rotateGenericOAuthAccountOn429(
   retryAfterHeader: string | null | undefined,
   now = Date.now(),
   requestedModelId?: string | null,
+  commit = true,
 ): string | null {
   return rotateGenericOAuthAccountOnRefusal(config, providerName, failedAccountId, "rate",
-    retryAfterHeader, now, requestedModelId);
+    retryAfterHeader, now, requestedModelId, undefined, commit);
 }
 
 /** Kiro adds account-scoped refusal classes; all other providers retain their 429 policy. */
@@ -507,6 +509,7 @@ export function rotateGenericOAuthAccountOnRefusal(
   now = Date.now(),
   requestedModelId?: string | null,
   monthlyCooldownMs?: number,
+  commit = true,
 ): string | null {
   if (kind === "other" || (providerName !== "kiro" && kind !== "rate")) return null;
   if (!isGenericOAuthFailoverEnabled(config, providerName)) return null;
@@ -566,8 +569,8 @@ export function rotateGenericOAuthAccountOnRefusal(
   }
   if (strategy === "round-robin") {
     // PICK here, not peek: the failure already happened and this answer is the one being used,
-    // so the ring genuinely advances.
-    return pickRoundRobinAccount(
+    // so the ring genuinely advances. A caller that only asks whether a sibling exists peeks.
+    return (commit ? pickRoundRobinAccount : peekRoundRobinAccount)(
       genericPoolKey(providerName),
       candidates,
       genericStickyLimit(config, providerName),

src/server/responses/core-combo.ts

@@ -52,14 +52,16 @@ import {
 import { hasUnreadableEncryptedAgentTask } from "./encrypted-payload";
 import { routeConcreteModel, comboRouteDecisionTrace } from "../../router";
 import { memoryModelRouteReason } from "./memory-models";
-import { poolAccountProviderLabel } from "../../providers/label";
+import { baseProviderLabel, poolAccountProviderLabel } from "../../providers/label";
 import { getAccountSet } from "../../oauth/store";
 import { configuredAnthropicInstance } from "../../providers/anthropic-instance";
+import { isAnthropicBillingFailure } from "../../providers/anthropic-billing-refusal";
 import {
   anthropicRoutingFor,
   formatAnthropicProviderForLog,
 } from "../../oauth/anthropic-routing";
-import { codexAccountLogLabel } from "../../codex/account-label";
+import { codexAccountLogLabel, oauthAccountLogLabel } from "../../codex/account-label";
+import { isGenericFailoverProvider, rotateGenericOAuthAccountOn429 } from "../../oauth/generic-account-failover";
 import { codexQuotaScopeForModel, getCodexQuotaHealthSnapshot } from "../../codex/routing";
 import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers";
 import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery";
@@ -135,6 +137,48 @@ function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId
     ? label : undefined;
 }
 
+/**
+ * The generic OAuth pool's counterpart of `cooledPoolAccountLabel`. Its 429 names one account
+ * through `accountLogLabel`, but the per-request account budget can end the pool's own rotation
+ * before the last refused account is recorded. Hand that account to the pool's rotator, which
+ * cools it under the pool's own Retry-After policy, and spare the target only when the pool still
+ * has an eligible sibling. The rotator only peeks round-robin here: nothing is sent to that sibling.
+ * A lone account or an all-cooled pool returns null and cools the target.
+ */
+function cooledGenericOAuthAccountLabel(
+  config: OcxConfig, providerName: string, modelId: string, label: string | undefined, retryAfter: string | undefined,
+): string | undefined {
+  const provider = config.providers[providerName];
+  if (!label || !provider || !isGenericFailoverProvider(providerName, provider)) return undefined;
+  const base = baseProviderLabel(providerName);
+  const matches = getAccountSet(providerName)?.accounts.filter(account =>
+    oauthAccountLogLabel(account.id, base) === label) ?? [];
+  return matches.length === 1
+    && rotateGenericOAuthAccountOn429(config, providerName, matches[0]!.id, retryAfter, Date.now(), modelId, false)
+    ? label : undefined;
+}
+
+/**
+ * A 402 credit-balance refusal names one Anthropic OAuth account -- the evidence the pool already
+ * accepts on a 403 -- but the pool rotates only on 401/403/429, so the 402 reaches the combo with
+ * that account still selectable. Cool it under the pool's entitlement backoff and spare the target
+ * only while a sibling is eligible; a lone account or an exhausted pool still cools the target.
+ */
+function creditRefusedAnthropicAccountLabel(
+  config: OcxConfig, providerName: string, modelId: string, label: string | undefined,
+  message: string, code: string | null | undefined,
+): string | undefined {
+  const instance = configuredAnthropicInstance(config, providerName);
+  if (!label || !instance || config.providers[providerName]?.authMode !== "oauth"
+    || !isAnthropicBillingFailure(402, message, code)) return undefined;
+  const routing = anthropicRoutingFor(instance);
+  const matches = getAccountSet(instance)?.accounts.filter(account =>
+    formatAnthropicProviderForLog(instance, account.id) === label) ?? [];
+  return matches.length === 1
+    && routing.recordAnthropicAccountRefusal(config, matches[0]!.id, 403, null)
+    && routing.getEligibleAnthropicAccounts(Date.now(), modelId).length > 0
+    ? label : undefined;
+}
 
 /**
  * True only for the Anthropic OAuth pool's OWN "all accounts cooled" 429. Its Retry-After is the
@@ -1135,12 +1179,15 @@ export async function executeComboResponses(
     const failedTargetKey = targetKey(pick.target);
     let failedTargetCooldownRecorded = false;
     // The dispatch rewrote this to name the pool account that actually served the turn.
-    const failedAccount = cooledPoolAccountLabel(
-      config,
-      pick.target.provider,
-      pick.target.model,
-      poolAccountProviderLabel(childLog.provider, pick.target.provider),
-    );
+    const poolAccountLabel = poolAccountProviderLabel(childLog.provider, pick.target.provider);
+    const failedAccount = failure.response.status === 402
+      ? creditRefusedAnthropicAccountLabel(
+        config, pick.target.provider, pick.target.model, poolAccountLabel, failure.classificationText, failure.upstreamCode,
+      )
+      : cooledPoolAccountLabel(config, pick.target.provider, pick.target.model, poolAccountLabel)
+      ?? (failure.response.status === 429 && failureCooldownScope === "target"
+      ? cooledGenericOAuthAccountLabel(config, pick.target.provider, pick.target.model, childLog.accountLogLabel, failure.retryAfter)
+      : undefined);
     // The target still cools on the pool's own refusal (the anti-hammer guard), but for the local
     // fallback rather than the pool's restated per-account Retry-After.
     const poolLocalRefusal = isAnthropicPoolLocalRefusal(

structure/runtime.md

@@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str
 
 Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`.
 
-`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged.
+`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan <window> quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged.
 
 ## Combo default effort precedence
 

tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts

@@ -182,3 +182,50 @@ test("a lone OAuth account without pool cooldown still cools the combo target",
   expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true);
   expect(sentTokens).toHaveLength(1);
 });
+
+/** The pool's own credit-balance wording (`isAccountRefusal`), on a 402. */
+function creditRefused(): Response {
+  return Response.json({ type: "error", error: {
+    type: "billing_error",
+    message: "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.",
+  } }, { status: 402 });
+}
+
+test("one account's 402 credit refusal does not cool the combo target while a sibling is eligible", async () => {
+  const ids = await seed(2);
+  const config = configFor(() => answered());
+  const anthropic = config.providers.anthropic as OcxProviderConfig & { fetch: typeof fetch };
+  // Only the first account is out of credit; the second still answers.
+  anthropic.fetch = (async (_input, init) => {
+    const token = String(new Headers(init?.headers).get("authorization"));
+    sentTokens.push(token);
+    return token === `Bearer ${credential(0).access}` ? creditRefused() : answered();
+  }) as typeof fetch;
+  const fallbackSends: string[] = [];
+  config.providers.fallback = {
+    adapter: "anthropic", baseUrl: "https://fallback.test", authMode: "key", apiKey: "synthetic-fallback-key",
+    models: [MODEL], fetch: (async () => { fallbackSends.push("B"); return answered(); }) as typeof fetch,
+  } as OcxProviderConfig;
+  config.combos = { pooled: { strategy: "failover", targets: [TARGET, { provider: "fallback", model: MODEL }] } };
+
+  expect((await post(config)).status).toBe(200);
+  expect(sentTokens).toEqual([`Bearer ${credential(0).access}`]);
+  expect(fallbackSends).toEqual(["B"]);
+  // The refusal names one account: the pool cools it, and the target stays usable for the other.
+  expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false);
+  expect(getAnthropicAccountHealthSnapshot(ids[0]!)).not.toBeNull();
+
+  sentTokens = [];
+  expect((await post(config)).status).toBe(200);
+  expect(sentTokens).toEqual([`Bearer ${credential(1).access}`]);
+  expect(fallbackSends).toEqual(["B"]);
+}, 20_000);
+
+test("402 credit refusals from every pool account still cool the combo target", async () => {
+  await seed(2);
+  const config = configFor(creditRefused);
+  // The first refusal leaves a sibling and spares the target; the second leaves none.
+  for (let attempt = 0; attempt < 2; attempt++) expect((await post(config)).status).toBe(402);
+  expect(sentTokens).toHaveLength(2);
+  expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true);
+}, 20_000);

tests/fixtures/test-layout-expected.json

@@ -661,7 +661,7 @@
   "gemini-inline.test.ts": "images", "gemini-web-search.test.ts": "adapters/google",
   "generic-session-affinity.test.ts": "oauth",
   "server-generic-oauth-affinity.test.ts": "server",
-  "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth",
+  "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth",
   "github-copilot-auto.test.ts": "providers/github-copilot",
   "github-copilot-auto-runtime.test.ts": "providers/github-copilot",
   "github-copilot-catalog-hints.test.ts": "providers/github-copilot",