fix: carry OAuth combo cooldown, account rotation and health evidence fixes

docs-site/src/content/docs/guides/combos.md

@@ -873,7 +873,10 @@ running opencodex instance that receives model requests.
 
 Every target is currently ineligible: for example, its provider is disabled, it is cooling down,
 it has already been attempted for this request, or an encrypted v2 task excludes it. Check target
-provider state and recent upstream errors. For cooldowns, follow an observed `Retry-After` value first;
+provider state and recent upstream errors. When no target was attempted, the error message lists
+why each target was skipped, for example
+`No available targets for combo: fast (openai/gpt-6: quota exhausted; anthropic/claude-opus-5-5: cooldown 42s)`.
+For cooldowns, follow an observed `Retry-After` value first;
 Codex reset headers also take precedence over `cooldownMs`.
 If neither upstream signal is usable, the configured `cooldownMs` applies, or the upstream fallback applies
 when it is unset (5 seconds for request-rate codes `1302`/`1305`, 10 minutes for a spent usage

scripts/test-layout/layout.json

@@ -943,6 +943,7 @@
     "gemini-web-search.test.ts": "adapters/google",
     "generic-session-affinity.test.ts": "oauth",
     "server-generic-oauth-affinity.test.ts": "server",
+    "generic-oauth-combo-account-cooldown.test.ts": "oauth",
     "generic-oauth-failover.test.ts": "oauth",
     "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot",
     "github-copilot-account-origin.test.ts": "providers/github-copilot",

src/combos/index.ts

@@ -22,6 +22,7 @@ export {
 export {
   advanceComboAfterFailure,
   clearComboSelectionState,
+  comboExclusionReasons,
   NoAvailableComboTargetsError,
   noteComboFailure,
   noteComboSuccess,

src/combos/resolve.ts

@@ -55,17 +55,49 @@ export class UnknownComboError extends Error {
 export class NoAvailableComboTargetsError extends Error {
   readonly code = "combo_unavailable";
 
-  constructor(readonly comboId: string) {
-    super(`No available targets for combo: ${comboId}`);
+  constructor(readonly comboId: string, readonly reasons = "") {
+    super(`No available targets for combo: ${comboId}${reasons ? ` (${reasons})` : ""}`);
     this.name = "NoAvailableComboTargetsError";
   }
 }
 
+function providerExclusionReason(config: OcxConfig, target: OcxComboTarget, now: number): string | undefined {
+  const provider = Object.hasOwn(config.providers, target.provider) ? config.providers[target.provider] : undefined;
+  if (!provider) return "provider missing";
+  if (provider.disabled === true) return "provider disabled";
+  return cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model)
+    ? "quota exhausted"
+    : undefined;
+}
+
 function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean {
-  if (!Object.hasOwn(config.providers, target.provider)) return false;
-  const provider = config.providers[target.provider];
-  if (!provider || provider.disabled === true) return false;
-  return !cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model);
+  return providerExclusionReason(config, target, now) === undefined;
+}
+
+/**
+ * Why each target of a combo cannot be picked right now, for a zero-attempt refusal. Only the
+ * configured provider/model and a coarse label are named, never account or request data.
+ */
+export function comboExclusionReasons(
+  config: OcxConfig,
+  comboId: string,
+  eligible?: (target: NormalizedComboTarget) => boolean,
+  now = Date.now(),
+): string {
+  return (getCombo(config, comboId)?.targets ?? []).flatMap(target => {
+    let reason = providerExclusionReason(config, target, now);
+    const cooldown = reason ? undefined : earliestComboCooldown(comboId, [target], now);
+    if (cooldown) reason = `cooldown ${Math.max(1, Math.ceil((cooldown.expiry - now) / 1000))}s`;
+    if (!reason && eligible) {
+      try {
+        if (!eligible(target)) reason = "ineligible for this request";
+      } catch {
+        // An eligibility check can exhaust the translator budget; the target still was not picked.
+        reason = "ineligible for this request";
+      }
+    }
+    return reason ? [`${targetKey(target)}: ${reason}`] : [];
+  }).join("; ");
 }
 
 /** Capture usable providers' quota cooldowns; request eligibility is deferred to exhaustion. */
@@ -411,9 +443,10 @@ export function advanceComboAfterFailure(
   // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks
   // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is
   // holding the target back, the target cooldown still does.
-  const accountScoped = options.cooldownScope === "target"
-    && options.status === 429
-    && options.failedAccount !== undefined;
+  // A 402 credit refusal is the provider-scoped twin: the caller names an account only after the
+  // pool has cooled it and still holds an eligible sibling.
+  const accountScoped = options.failedAccount !== undefined
+    && (options.status === 429 ? options.cooldownScope === "target" : options.status === 402);
   // "none" records no cooldown at all: the failure described the request, not the target, so
   // the target must stay immediately selectable for the next (differently shaped) request.
   if (options.cooldownScope !== "none" && !accountScoped) {
@@ -615,6 +648,6 @@ export function tryPickComboModel(config: OcxConfig, modelId: string, preview =
   if (!comboId) return null;
   if (!getCombo(config, comboId)) throw new UnknownComboError(comboId);
   const picked = pickComboTarget(config, comboId, { preview });
-  if (!picked) throw new NoAvailableComboTargetsError(comboId);
+  if (!picked) throw new NoAvailableComboTargetsError(comboId, comboExclusionReasons(config, comboId));
   return picked;
 }

src/oauth/anthropic-account-refusal.ts

@@ -31,6 +31,22 @@ export function bindAnthropicRefusalCredentialForSend(response: Response, owner:
   responseCredentials.set(response, Object.freeze({ ...owner, providerAccountUuid, checkProviderUuid: arguments.length >= 3 }));
 }
 
+/** Preserve the original send proof across internal error normalization, without renewing it. */
+export function carryAnthropicRefusalCredential(source: Response, target: Response): void {
+  const sent = responseCredentials.get(source);
+  if (sent) responseCredentials.set(target, sent);
+}
+
+/** Recheck after body consumption: a late refusal cannot acquire a replacement login's authority. */
+export function anthropicRefusalCredentialIsCurrent(response: Response, instance: AnthropicInstanceId, accountId: string): boolean {
+  const sent = responseCredentials.get(response);
+  if (!sent || sent.provider !== instance || sent.accountId !== accountId
+    || !anthropicPhysicalSendOwnershipIsCurrent(sent)) return false;
+  const row = getAccountCredentialWithStatus(instance, accountId);
+  return !!row && !row.paused && !row.needsReauth
+    && (!sent.checkProviderUuid || row.credential.accountId === sent.providerAccountUuid);
+}
+
 async function isRevokedOAuthToken(response: Response, signal?: AbortSignal): Promise<boolean> {
   if (response.status !== 401) return false;
   try {
@@ -59,9 +75,11 @@ async function isAccountRefusal(response: Response, signal?: AbortSignal): Promi
     // Whole-message patterns: request-policy/model/resource refusals and quoted diagnostics
     // must not acquire account authority merely by containing entitlement keywords.
     if ("code" in error && error.code != null
-      && (typeof error.code !== "string" || !["subscription_required", "insufficient_quota", "permission_denied"].includes(error.code))) return false;
+      && (typeof error.code !== "string" || !["subscription_required", "insufficient_quota", "permission_denied", "oauth_org_not_allowed"].includes(error.code))) return false;
+    // The organization disabled subscription access: an account-scoped verdict, cooled like its siblings.
+    if (error.type === "oauth_org_not_allowed" || ("code" in error && error.code === "oauth_org_not_allowed")) return true;
     const message = error.message.trim();
-    if (error.type === "permission_error" && /^(?:Your account does not have access to Claude Code|Your (?:Claude )?subscription (?:has expired|is (?:expired|inactive))|Your account does not have an active subscription)[.!]?$/i.test(message)) return true;
+    if (error.type === "permission_error" && /^(?:Your account does not have access to Claude Code|Your (?:Claude )?subscription (?:has expired|is (?:expired|inactive))|Your account does not have an active subscription|(?:Your|This) organization has disabled Claude subscription access for Claude Code(?:\. Use an Anthropic API key instead, or ask your admin to enable access)?)[.!]?$/i.test(message)) return true;
     return (error.type === "billing_error" || error.type === "permission_error")
       && /^Your credit balance is too low to access the Anthropic API\.(?: Please go to Plans & Billing to upgrade or purchase credits\.)?$/i.test(message);
   } catch {

src/routing/health.ts

@@ -94,7 +94,6 @@ interface HealthSample {
 }
 
 interface HealthRow extends HealthSample {
-  attemptCount?: number;
   rowJson?: string | null;
 }
 
@@ -103,33 +102,53 @@ interface HealthRow extends HealthSample {
  * Combo/failover requests store each upstream try in `entry.attempts` while
  * the top-level row records the final outcome; a provider/model that failed
  * as a non-final attempt must still contribute its own health samples.
+ *
+ * Only physical sends are evidence: an attempt answered locally or refused
+ * before it left the proxy says nothing about the target. The row's terminal
+ * facts belong to its last physical send. Returns `undefined` when the row
+ * itself is the sample (no attempts, a single attempt, or none named for this
+ * candidate) and `[]` when the row sent nothing upstream at all.
  */
 function attemptSamplesFor(
-  row: Pick<HealthRow, "timestamp" | "attemptCount" | "rowJson">,
+  row: Pick<HealthRow, "timestamp" | "rowJson">,
   provider: string,
   model: string,
-): HealthSample[] {
-  if (!row.rowJson || (row.attemptCount ?? 1) <= 1) return [];
+): HealthSample[] | undefined {
+  if (!row.rowJson) return undefined;
   try {
-    const parsed = JSON.parse(row.rowJson) as { attempts?: unknown };
-    if (!Array.isArray(parsed.attempts)) return [];
+    const parsed = JSON.parse(row.rowJson) as {
+      attempts?: unknown;
+      closeReason?: unknown;
+      terminalStatus?: unknown;
+      failureCause?: unknown;
+    };
+    // A local refusal logged before any attempt began never reached a target.
+    if (!Array.isArray(parsed.attempts)) return parsed.failureCause === "local-refusal" ? [] : undefined;
+    const attempts = parsed.attempts.filter(
+      (attempt): attempt is Record<string, unknown> =>
+        !!attempt && typeof attempt === "object" && !Array.isArray(attempt),
+    );
+    const sent = attempts.filter(attempt => attempt.locallyAnswered !== true && attempt.sendCount !== 0);
+    if (sent.length === 0) return [];
+    const mine = attempts.filter(attempt => attempt.provider === provider && attempt.model === model);
+    if (attempts.length <= 1 || mine.length === 0) return undefined;
+    const finalSend = sent.at(-1);
     const samples: HealthSample[] = [];
-    for (const attempt of parsed.attempts) {
-      if (!attempt || typeof attempt !== "object" || Array.isArray(attempt)) continue;
-      const record = attempt as Record<string, unknown>;
-      if (record.provider !== provider || record.model !== model) continue;
+    for (const record of mine) {
+      if (!sent.includes(record)) continue;
       if (typeof record.status !== "number" || typeof record.durationMs !== "number") continue;
+      const isFinal = record === finalSend;
       samples.push({
         status: record.status,
-        closeReason: null,
-        terminalStatus: null,
+        closeReason: isFinal && typeof parsed.closeReason === "string" ? parsed.closeReason : null,
+        terminalStatus: isFinal && typeof parsed.terminalStatus === "string" ? parsed.terminalStatus : null,
         durationMs: record.durationMs,
         timestamp: row.timestamp,
       });
     }
     return samples;
   } catch {
-    return [];
+    return undefined;
   }
 }
 
@@ -243,8 +262,7 @@ function computeHistoricalHealthEvidence(
     }
     const rows = handle.query(
       `SELECT status, close_reason AS closeReason, terminal_status AS terminalStatus,
-              duration_ms AS durationMs, timestamp,
-              attempt_count AS attemptCount, row_json AS rowJson
+              duration_ms AS durationMs, timestamp, row_json AS rowJson
        FROM requests WHERE ${where.join(" AND ")}
        ORDER BY timestamp DESC LIMIT ?`,
     ).all(...values, HEALTH_MAX_SAMPLES) as HealthRow[];
@@ -254,7 +272,7 @@ function computeHistoricalHealthEvidence(
     // rows that cannot contribute samples for this candidate.
     const escapeLike = (value: string): string => value.replace(/[\\%_]/g, match => `\\${match}`);
     const attemptRows = handle.query(
-      `SELECT timestamp, attempt_count AS attemptCount, row_json AS rowJson
+      `SELECT timestamp, row_json AS rowJson
        FROM requests WHERE timestamp >= ? AND attempt_count > 1
          AND row_json LIKE ? ESCAPE '\\'
          AND row_json LIKE ? ESCAPE '\\'
@@ -267,17 +285,14 @@ function computeHistoricalHealthEvidence(
       input.provider,
       input.model,
       HEALTH_MAX_SAMPLES,
-    ) as Array<
-      Pick<HealthRow, "timestamp" | "attemptCount" | "rowJson">
-    >;
+    ) as Array<Pick<HealthRow, "timestamp" | "rowJson">>;
 
     const samples: HealthSample[] = [];
     for (const row of rows) {
-      const attemptSamples = attemptSamplesFor(row, input.provider, input.model);
-      samples.push(...(attemptSamples.length > 0 ? attemptSamples : [row]));
+      samples.push(...(attemptSamplesFor(row, input.provider, input.model) ?? [row]));
     }
     for (const row of attemptRows) {
-      samples.push(...attemptSamplesFor(row, input.provider, input.model));
+      samples.push(...(attemptSamplesFor(row, input.provider, input.model) ?? []));
     }
     // Newest first for the consecutive-failure walk; attempt samples inherit
     // their row's timestamp.

src/server/responses/adapter-continuation.ts

@@ -440,9 +440,20 @@ export function createAdapterContinuations(
               resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
               config.cacheRetention,
             );
+            // Rebind both the continuation clone and the outer request that persists the response.
+            for (const requestParsed of [nextParsed, parsed]) {
+              bindRouteReasoningReplayScope({
+                parsed: requestParsed,
+                providerName: route.providerName,
+                provider: route.provider,
+                adapterName: transportState.activeAdapter.name,
+                oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
+              });
+            }
             sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
             recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
-            nextContinuationRecoveryKind = response.status === 401 ? "oauth-401" : "anthropic-oauth-429";
+            nextContinuationRecoveryKind = response.status === 401 ? "oauth-401"
+              : response.status === 403 ? "oauth-account-403" : "anthropic-oauth-429";
             continue;
           } catch {
             // fall through to emit continuation error below

src/server/responses/adapter-dispatch.ts

@@ -1346,9 +1346,17 @@ export async function prepareAdapterExchange(
             resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy),
             config.cacheRetention,
           );
+          bindRouteReasoningReplayScope({
+            parsed,
+            providerName: route.providerName,
+            provider: route.provider,
+            adapterName: transportState.activeAdapter.name,
+            oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot,
+          });
           sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel);
           recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name);
-         const result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401" : "anthropic-oauth-429");
+         const result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401"
+           : upstreamResponse.status === 403 ? "oauth-account-403" : "anthropic-oauth-429");
          if ("failed" in result) return result.failed;
          upstreamResponse = result;
           if (isNonReplayableResponse(upstreamResponse)) continue recovery;

src/server/responses/context-overflow.ts

@@ -17,21 +17,25 @@ export function lateAttributionLimitResponse(): Response {
 export const PROVIDER_INPUT_TOO_LARGE_MESSAGE =
   "The provider rejected this turn because its input exceeds the provider size or context limit. Reduce the current input or compact the conversation before retrying.";
 
-/** Preserve non-streaming HTTP failure semantics without exposing an upstream body. */
-export function jsonContextOverflowResponse(): Response {
+/**
+ * Preserve non-streaming HTTP failure semantics without exposing an upstream body. `message` is
+ * replaced only for the proxy's own wording, never with upstream text: when every combo target
+ * was refused by local input admission, no provider saw the turn and saying one did is false.
+ */
+export function jsonContextOverflowResponse(message = PROVIDER_INPUT_TOO_LARGE_MESSAGE): Response {
   return Response.json({
     error: {
-      message: PROVIDER_INPUT_TOO_LARGE_MESSAGE,
+      message,
       type: "invalid_request_error",
       code: "context_length_exceeded",
     },
   }, { status: 413, headers: { "Cache-Control": "no-store" } });
 }
 
-async function* contextOverflowEvents(): AsyncGenerator<AdapterEvent> {
+async function* contextOverflowEvents(message: string): AsyncGenerator<AdapterEvent> {
   yield {
     type: "error",
-    message: PROVIDER_INPUT_TOO_LARGE_MESSAGE,
+    message,
     status: 413,
     errorType: "invalid_request_error",
     code: "context_length_exceeded",
@@ -51,9 +55,10 @@ async function* contextOverflowEvents(): AsyncGenerator<AdapterEvent> {
 export function streamingContextOverflowResponse(
   modelId: string,
   translatorBudget: TranslatorBudget,
+  message = PROVIDER_INPUT_TOO_LARGE_MESSAGE,
 ): Response {
   return new Response(bridgeToResponsesSSE(
-    contextOverflowEvents(),
+    contextOverflowEvents(message),
     modelId,
     undefined,
     undefined,