diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 942feb374b4..7673c56b727 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -873,7 +873,10 @@ running opencodex instance that receives model requests. Every target is currently ineligible: for example, its provider is disabled, it is cooling down, it has already been attempted for this request, or an encrypted v2 task excludes it. Check target -provider state and recent upstream errors. For cooldowns, follow an observed `Retry-After` value first; +provider state and recent upstream errors. When no target was attempted, the error message lists +why each target was skipped, for example +`No available targets for combo: fast (openai/gpt-6: quota exhausted; anthropic/claude-opus-5-5: cooldown 42s)`. +For cooldowns, follow an observed `Retry-After` value first; Codex reset headers also take precedence over `cooldownMs`. If neither upstream signal is usable, the configured `cooldownMs` applies, or the upstream fallback applies when it is unset (5 seconds for request-rate codes `1302`/`1305`, 10 minutes for a spent usage diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c077bccfac3..a3fccb6225f 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -943,6 +943,7 @@ "gemini-web-search.test.ts": "adapters/google", "generic-session-affinity.test.ts": "oauth", "server-generic-oauth-affinity.test.ts": "server", + "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot", "github-copilot-account-origin.test.ts": "providers/github-copilot", diff --git a/src/combos/index.ts b/src/combos/index.ts index 6db073f0a79..0b22c99d814 100644 --- a/src/combos/index.ts +++ b/src/combos/index.ts @@ -22,6 +22,7 @@ export { export { advanceComboAfterFailure, clearComboSelectionState, + comboExclusionReasons, NoAvailableComboTargetsError, noteComboFailure, noteComboSuccess, diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index a1c07fc1ab7..6f05123717a 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -55,17 +55,49 @@ export class UnknownComboError extends Error { export class NoAvailableComboTargetsError extends Error { readonly code = "combo_unavailable"; - constructor(readonly comboId: string) { - super(`No available targets for combo: ${comboId}`); + constructor(readonly comboId: string, readonly reasons = "") { + super(`No available targets for combo: ${comboId}${reasons ? ` (${reasons})` : ""}`); this.name = "NoAvailableComboTargetsError"; } } +function providerExclusionReason(config: OcxConfig, target: OcxComboTarget, now: number): string | undefined { + const provider = Object.hasOwn(config.providers, target.provider) ? config.providers[target.provider] : undefined; + if (!provider) return "provider missing"; + if (provider.disabled === true) return "provider disabled"; + return cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model) + ? "quota exhausted" + : undefined; +} + function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget, now: number): boolean { - if (!Object.hasOwn(config.providers, target.provider)) return false; - const provider = config.providers[target.provider]; - if (!provider || provider.disabled === true) return false; - return !cachedProviderQuotaIsExhausted(getCachedProviderRoutingQuota(target.provider, provider, now), now, target.model); + return providerExclusionReason(config, target, now) === undefined; +} + +/** + * Why each target of a combo cannot be picked right now, for a zero-attempt refusal. Only the + * configured provider/model and a coarse label are named, never account or request data. + */ +export function comboExclusionReasons( + config: OcxConfig, + comboId: string, + eligible?: (target: NormalizedComboTarget) => boolean, + now = Date.now(), +): string { + return (getCombo(config, comboId)?.targets ?? []).flatMap(target => { + let reason = providerExclusionReason(config, target, now); + const cooldown = reason ? undefined : earliestComboCooldown(comboId, [target], now); + if (cooldown) reason = `cooldown ${Math.max(1, Math.ceil((cooldown.expiry - now) / 1000))}s`; + if (!reason && eligible) { + try { + if (!eligible(target)) reason = "ineligible for this request"; + } catch { + // An eligibility check can exhaust the translator budget; the target still was not picked. + reason = "ineligible for this request"; + } + } + return reason ? [`${targetKey(target)}: ${reason}`] : []; + }).join("; "); } /** Capture usable providers' quota cooldowns; request eligibility is deferred to exhaustion. */ @@ -411,9 +443,10 @@ export function advanceComboAfterFailure( // including the pool's OWN "all accounts are cooled" 429, which refuses locally before it picks // one. That is what keeps this from trading a blackhole for a hammer: when nothing else is // holding the target back, the target cooldown still does. - const accountScoped = options.cooldownScope === "target" - && options.status === 429 - && options.failedAccount !== undefined; + // A 402 credit refusal is the provider-scoped twin: the caller names an account only after the + // pool has cooled it and still holds an eligible sibling. + const accountScoped = options.failedAccount !== undefined + && (options.status === 429 ? options.cooldownScope === "target" : options.status === 402); // "none" records no cooldown at all: the failure described the request, not the target, so // the target must stay immediately selectable for the next (differently shaped) request. if (options.cooldownScope !== "none" && !accountScoped) { @@ -615,6 +648,6 @@ export function tryPickComboModel(config: OcxConfig, modelId: string, preview = if (!comboId) return null; if (!getCombo(config, comboId)) throw new UnknownComboError(comboId); const picked = pickComboTarget(config, comboId, { preview }); - if (!picked) throw new NoAvailableComboTargetsError(comboId); + if (!picked) throw new NoAvailableComboTargetsError(comboId, comboExclusionReasons(config, comboId)); return picked; } diff --git a/src/oauth/anthropic-account-refusal.ts b/src/oauth/anthropic-account-refusal.ts index 63d4d5790a4..0885e95ffa9 100644 --- a/src/oauth/anthropic-account-refusal.ts +++ b/src/oauth/anthropic-account-refusal.ts @@ -31,6 +31,22 @@ export function bindAnthropicRefusalCredentialForSend(response: Response, owner: responseCredentials.set(response, Object.freeze({ ...owner, providerAccountUuid, checkProviderUuid: arguments.length >= 3 })); } +/** Preserve the original send proof across internal error normalization, without renewing it. */ +export function carryAnthropicRefusalCredential(source: Response, target: Response): void { + const sent = responseCredentials.get(source); + if (sent) responseCredentials.set(target, sent); +} + +/** Recheck after body consumption: a late refusal cannot acquire a replacement login's authority. */ +export function anthropicRefusalCredentialIsCurrent(response: Response, instance: AnthropicInstanceId, accountId: string): boolean { + const sent = responseCredentials.get(response); + if (!sent || sent.provider !== instance || sent.accountId !== accountId + || !anthropicPhysicalSendOwnershipIsCurrent(sent)) return false; + const row = getAccountCredentialWithStatus(instance, accountId); + return !!row && !row.paused && !row.needsReauth + && (!sent.checkProviderUuid || row.credential.accountId === sent.providerAccountUuid); +} + async function isRevokedOAuthToken(response: Response, signal?: AbortSignal): Promise { if (response.status !== 401) return false; try { @@ -59,9 +75,11 @@ async function isAccountRefusal(response: Response, signal?: AbortSignal): Promi // Whole-message patterns: request-policy/model/resource refusals and quoted diagnostics // must not acquire account authority merely by containing entitlement keywords. if ("code" in error && error.code != null - && (typeof error.code !== "string" || !["subscription_required", "insufficient_quota", "permission_denied"].includes(error.code))) return false; + && (typeof error.code !== "string" || !["subscription_required", "insufficient_quota", "permission_denied", "oauth_org_not_allowed"].includes(error.code))) return false; + // The organization disabled subscription access: an account-scoped verdict, cooled like its siblings. + if (error.type === "oauth_org_not_allowed" || ("code" in error && error.code === "oauth_org_not_allowed")) return true; const message = error.message.trim(); - if (error.type === "permission_error" && /^(?:Your account does not have access to Claude Code|Your (?:Claude )?subscription (?:has expired|is (?:expired|inactive))|Your account does not have an active subscription)[.!]?$/i.test(message)) return true; + if (error.type === "permission_error" && /^(?:Your account does not have access to Claude Code|Your (?:Claude )?subscription (?:has expired|is (?:expired|inactive))|Your account does not have an active subscription|(?:Your|This) organization has disabled Claude subscription access for Claude Code(?:\. Use an Anthropic API key instead, or ask your admin to enable access)?)[.!]?$/i.test(message)) return true; return (error.type === "billing_error" || error.type === "permission_error") && /^Your credit balance is too low to access the Anthropic API\.(?: Please go to Plans & Billing to upgrade or purchase credits\.)?$/i.test(message); } catch { diff --git a/src/routing/health.ts b/src/routing/health.ts index 9f3f2115d6e..ceab2e1b9a1 100644 --- a/src/routing/health.ts +++ b/src/routing/health.ts @@ -94,7 +94,6 @@ interface HealthSample { } interface HealthRow extends HealthSample { - attemptCount?: number; rowJson?: string | null; } @@ -103,33 +102,53 @@ interface HealthRow extends HealthSample { * Combo/failover requests store each upstream try in `entry.attempts` while * the top-level row records the final outcome; a provider/model that failed * as a non-final attempt must still contribute its own health samples. + * + * Only physical sends are evidence: an attempt answered locally or refused + * before it left the proxy says nothing about the target. The row's terminal + * facts belong to its last physical send. Returns `undefined` when the row + * itself is the sample (no attempts, a single attempt, or none named for this + * candidate) and `[]` when the row sent nothing upstream at all. */ function attemptSamplesFor( - row: Pick, + row: Pick, provider: string, model: string, -): HealthSample[] { - if (!row.rowJson || (row.attemptCount ?? 1) <= 1) return []; +): HealthSample[] | undefined { + if (!row.rowJson) return undefined; try { - const parsed = JSON.parse(row.rowJson) as { attempts?: unknown }; - if (!Array.isArray(parsed.attempts)) return []; + const parsed = JSON.parse(row.rowJson) as { + attempts?: unknown; + closeReason?: unknown; + terminalStatus?: unknown; + failureCause?: unknown; + }; + // A local refusal logged before any attempt began never reached a target. + if (!Array.isArray(parsed.attempts)) return parsed.failureCause === "local-refusal" ? [] : undefined; + const attempts = parsed.attempts.filter( + (attempt): attempt is Record => + !!attempt && typeof attempt === "object" && !Array.isArray(attempt), + ); + const sent = attempts.filter(attempt => attempt.locallyAnswered !== true && attempt.sendCount !== 0); + if (sent.length === 0) return []; + const mine = attempts.filter(attempt => attempt.provider === provider && attempt.model === model); + if (attempts.length <= 1 || mine.length === 0) return undefined; + const finalSend = sent.at(-1); const samples: HealthSample[] = []; - for (const attempt of parsed.attempts) { - if (!attempt || typeof attempt !== "object" || Array.isArray(attempt)) continue; - const record = attempt as Record; - if (record.provider !== provider || record.model !== model) continue; + for (const record of mine) { + if (!sent.includes(record)) continue; if (typeof record.status !== "number" || typeof record.durationMs !== "number") continue; + const isFinal = record === finalSend; samples.push({ status: record.status, - closeReason: null, - terminalStatus: null, + closeReason: isFinal && typeof parsed.closeReason === "string" ? parsed.closeReason : null, + terminalStatus: isFinal && typeof parsed.terminalStatus === "string" ? parsed.terminalStatus : null, durationMs: record.durationMs, timestamp: row.timestamp, }); } return samples; } catch { - return []; + return undefined; } } @@ -243,8 +262,7 @@ function computeHistoricalHealthEvidence( } const rows = handle.query( `SELECT status, close_reason AS closeReason, terminal_status AS terminalStatus, - duration_ms AS durationMs, timestamp, - attempt_count AS attemptCount, row_json AS rowJson + duration_ms AS durationMs, timestamp, row_json AS rowJson FROM requests WHERE ${where.join(" AND ")} ORDER BY timestamp DESC LIMIT ?`, ).all(...values, HEALTH_MAX_SAMPLES) as HealthRow[]; @@ -254,7 +272,7 @@ function computeHistoricalHealthEvidence( // rows that cannot contribute samples for this candidate. const escapeLike = (value: string): string => value.replace(/[\\%_]/g, match => `\\${match}`); const attemptRows = handle.query( - `SELECT timestamp, attempt_count AS attemptCount, row_json AS rowJson + `SELECT timestamp, row_json AS rowJson FROM requests WHERE timestamp >= ? AND attempt_count > 1 AND row_json LIKE ? ESCAPE '\\' AND row_json LIKE ? ESCAPE '\\' @@ -267,17 +285,14 @@ function computeHistoricalHealthEvidence( input.provider, input.model, HEALTH_MAX_SAMPLES, - ) as Array< - Pick - >; + ) as Array>; const samples: HealthSample[] = []; for (const row of rows) { - const attemptSamples = attemptSamplesFor(row, input.provider, input.model); - samples.push(...(attemptSamples.length > 0 ? attemptSamples : [row])); + samples.push(...(attemptSamplesFor(row, input.provider, input.model) ?? [row])); } for (const row of attemptRows) { - samples.push(...attemptSamplesFor(row, input.provider, input.model)); + samples.push(...(attemptSamplesFor(row, input.provider, input.model) ?? [])); } // Newest first for the consecutive-failure walk; attempt samples inherit // their row's timestamp. diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 7ce56410fbc..030d4b22185 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -440,9 +440,20 @@ export function createAdapterContinuations( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy), config.cacheRetention, ); + // Rebind both the continuation clone and the outer request that persists the response. + for (const requestParsed of [nextParsed, parsed]) { + bindRouteReasoningReplayScope({ + parsed: requestParsed, + providerName: route.providerName, + provider: route.provider, + adapterName: transportState.activeAdapter.name, + oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot, + }); + } sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); - nextContinuationRecoveryKind = response.status === 401 ? "oauth-401" : "anthropic-oauth-429"; + nextContinuationRecoveryKind = response.status === 401 ? "oauth-401" + : response.status === 403 ? "oauth-account-403" : "anthropic-oauth-429"; continue; } catch { // fall through to emit continuation error below diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 6f418529a3f..afb473ab583 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -1346,9 +1346,17 @@ export async function prepareAdapterExchange( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy), config.cacheRetention, ); + bindRouteReasoningReplayScope({ + parsed, + providerName: route.providerName, + provider: route.provider, + adapterName: transportState.activeAdapter.name, + oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot, + }); sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); - const result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401" : "anthropic-oauth-429"); + const result = await rebuildAndRefetch(upstreamResponse.status === 401 ? "oauth-401" + : upstreamResponse.status === 403 ? "oauth-account-403" : "anthropic-oauth-429"); if ("failed" in result) return result.failed; upstreamResponse = result; if (isNonReplayableResponse(upstreamResponse)) continue recovery; diff --git a/src/server/responses/context-overflow.ts b/src/server/responses/context-overflow.ts index 2cfc1f96175..e90b736e37f 100644 --- a/src/server/responses/context-overflow.ts +++ b/src/server/responses/context-overflow.ts @@ -17,21 +17,25 @@ export function lateAttributionLimitResponse(): Response { export const PROVIDER_INPUT_TOO_LARGE_MESSAGE = "The provider rejected this turn because its input exceeds the provider size or context limit. Reduce the current input or compact the conversation before retrying."; -/** Preserve non-streaming HTTP failure semantics without exposing an upstream body. */ -export function jsonContextOverflowResponse(): Response { +/** + * Preserve non-streaming HTTP failure semantics without exposing an upstream body. `message` is + * replaced only for the proxy's own wording, never with upstream text: when every combo target + * was refused by local input admission, no provider saw the turn and saying one did is false. + */ +export function jsonContextOverflowResponse(message = PROVIDER_INPUT_TOO_LARGE_MESSAGE): Response { return Response.json({ error: { - message: PROVIDER_INPUT_TOO_LARGE_MESSAGE, + message, type: "invalid_request_error", code: "context_length_exceeded", }, }, { status: 413, headers: { "Cache-Control": "no-store" } }); } -async function* contextOverflowEvents(): AsyncGenerator { +async function* contextOverflowEvents(message: string): AsyncGenerator { yield { type: "error", - message: PROVIDER_INPUT_TOO_LARGE_MESSAGE, + message, status: 413, errorType: "invalid_request_error", code: "context_length_exceeded", @@ -51,9 +55,10 @@ async function* contextOverflowEvents(): AsyncGenerator { export function streamingContextOverflowResponse( modelId: string, translatorBudget: TranslatorBudget, + message = PROVIDER_INPUT_TOO_LARGE_MESSAGE, ): Response { return new Response(bridgeToResponsesSSE( - contextOverflowEvents(), + contextOverflowEvents(message), modelId, undefined, undefined, diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 17071ba24d2..bad576636da 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -32,6 +32,7 @@ import { type CodexWsPreOutputRefusalHold, } from "./codex-ws-wire"; import type { RequestLogContext } from "../request-log"; +import type { AttemptRecoveryKind } from "../../usage/log"; import type { DataPlaneAdmission } from "../auth-cors"; import type { InboundWire } from "../../providers/registry"; import type { BunRuntimeGateInput } from "./ws-upstream"; @@ -971,6 +972,10 @@ export async function retryCodexPoolOnAlternateAccount( // What this diff does fix is that the rungs are now CHARGED instead of free. const maxRetrySends = retrySameConfirmedAccount ? 7 : 1; let retrySendCount = 0; + // The gated-model 400 has no roster kind; it stays unattributed as before. + const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 || outcomeStatus === 402 + ? "oauth-account-429" + : isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined; let upstreamResponse: Response; try { while (true) { @@ -1005,7 +1010,10 @@ export async function retryCodexPoolOnAlternateAccount( // The move is a physical send like any other, so the root workflow is charged too. chargeWorkflowSends(args.options.workflowRootId, 1); } - noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate); + // A move seals the first account's attempt here. Give it the status that account actually + // answered, not the 502 the split infers, and name why the second send exists. + if (logCtx.activeAttempt && logCtx.activeAttempt.status < 100) logCtx.activeAttempt.status = firstResponse.status; + noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate, moveRecovery); try { upstreamResponse = await fetchWithHeaderTimeout( request.url, diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index 90542a06cf7..ebebcf56aeb 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -3,6 +3,7 @@ import { parseRetryAfterMs } from "../../combos"; import type { ConsumedComboFailure, HandleResponsesOptions } from "./core-options"; import type { OcxUsage } from "../../types"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +import { carryAnthropicRefusalCredential } from "../../oauth/anthropic-account-refusal"; import { codexQuotaFailureMessage, codexQuotaOutcomeMeta } from "./core-codex-account"; import { isRateLimitOrQuotaFailureMessage, @@ -125,6 +126,7 @@ export async function consumeComboFailure( }, ); if (nonReplayable) markResponseNonReplayable(failureResponse); + carryAnthropicRefusalCredential(response, failureResponse); return { response: failureResponse, ...(nonReplayable ? { nonReplayable: true } : {}), diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index f4714f0c9b7..3104433d4c7 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -31,6 +31,7 @@ import { comboDefaultEffort, isComboTargetInCooldown, snapshotComboQuotaCooldowns, + comboExclusionReasons, noteComboSuccess, comboFailureDecision, advanceComboAfterFailure, @@ -52,14 +53,17 @@ import { import { hasUnreadableEncryptedAgentTask } from "./encrypted-payload"; import { routeConcreteModel, comboRouteDecisionTrace } from "../../router"; import { memoryModelRouteReason } from "./memory-models"; -import { poolAccountProviderLabel } from "../../providers/label"; +import { baseProviderLabel, poolAccountProviderLabel } from "../../providers/label"; import { getAccountSet } from "../../oauth/store"; +import { anthropicRefusalCredentialIsCurrent } from "../../oauth/anthropic-account-refusal"; import { configuredAnthropicInstance } from "../../providers/anthropic-instance"; +import { isAnthropicBillingFailure } from "../../providers/anthropic-billing-refusal"; import { anthropicRoutingFor, formatAnthropicProviderForLog, } from "../../oauth/anthropic-routing"; -import { codexAccountLogLabel } from "../../codex/account-label"; +import { codexAccountLogLabel, oauthAccountLogLabel } from "../../codex/account-label"; +import { isGenericFailoverProvider, rotateGenericOAuthAccountOn429 } from "../../oauth/generic-account-failover"; import { codexQuotaScopeForModel, getCodexQuotaHealthSnapshot } from "../../codex/routing"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery"; @@ -135,6 +139,48 @@ function cooledPoolAccountLabel(config: OcxConfig, providerName: string, modelId ? label : undefined; } +/** + * The generic OAuth pool's counterpart of `cooledPoolAccountLabel`. Its 429 names one account + * through `accountLogLabel`, but the per-request account budget can end the pool's own rotation + * before the last refused account is recorded. Hand that account to the pool's rotator, which + * cools it under the pool's own Retry-After policy, and spare the target only when the pool still + * has an eligible sibling. A lone account or an all-cooled pool returns null and cools the target. + */ +function cooledGenericOAuthAccountLabel( + config: OcxConfig, providerName: string, modelId: string, label: string | undefined, retryAfter: string | undefined, +): string | undefined { + const provider = config.providers[providerName]; + if (!label || !provider || !isGenericFailoverProvider(providerName, provider)) return undefined; + const base = baseProviderLabel(providerName); + const matches = getAccountSet(providerName)?.accounts.filter(account => + oauthAccountLogLabel(account.id, base) === label) ?? []; + return matches.length === 1 + && rotateGenericOAuthAccountOn429(config, providerName, matches[0]!.id, retryAfter, Date.now(), modelId) + ? label : undefined; +} + +/** + * A 402 credit-balance refusal names one Anthropic OAuth account -- the evidence the pool already + * accepts on a 403 -- but the pool rotates only on 401/403/429, so the 402 reaches the combo with + * that account still selectable. Cool it under the pool's entitlement backoff and spare the target + * only while a sibling is eligible; a lone account or an exhausted pool still cools the target. + */ +function creditRefusedAnthropicAccountLabel( + config: OcxConfig, providerName: string, modelId: string, label: string | undefined, + message: string, code: string | null | undefined, response: Response, +): string | undefined { + const instance = configuredAnthropicInstance(config, providerName); + if (!label || !instance || config.providers[providerName]?.authMode !== "oauth" + || !isAnthropicBillingFailure(402, message, code)) return undefined; + const routing = anthropicRoutingFor(instance); + const matches = getAccountSet(instance)?.accounts.filter(account => + formatAnthropicProviderForLog(instance, account.id) === label) ?? []; + return matches.length === 1 + && anthropicRefusalCredentialIsCurrent(response, instance, matches[0]!.id) + && routing.recordAnthropicAccountRefusal(config, matches[0]!.id, 403, null) + && routing.getEligibleAnthropicAccounts(Date.now(), modelId).length > 0 + ? label : undefined; +} /** * True only for the Anthropic OAuth pool's OWN "all accounts cooled" 429. Its Retry-After is the @@ -251,6 +297,30 @@ interface JevComboChoice { candidate: JevCandidate; } +/** A target's JEV effort choices, or undefined when an explicit allowlist has none still supported. */ +function jevChoiceEfforts( + config: OcxConfig, + target: NonNullable>["targets"][number], +): OcxComboDefaultEffort[] | undefined { + let ladder: string[] | undefined; + try { + const route = routeConcreteModel(config, targetKey(target)); + ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId }); + } catch { + // Preserve the existing routing-failure surface. Unknown capability becomes the explicit + // no-effort choice rather than broadening JEV's effort allowlist. + ladder = undefined; + } + const supportedEfforts = (ladder ?? []).filter(isCodexReasoningEffort) as OcxComboDefaultEffort[]; + const configuredEfforts = target.reasoningEfforts; + const reasoningEfforts = configuredEfforts === undefined + ? supportedEfforts + : configuredEfforts.filter(effort => supportedEfforts.includes(effort)); + // An explicit allowlist is restrictive. If catalog capabilities drift until no configured + // effort remains supported, omit the target instead of silently broadening JEV's choices. + return configuredEfforts !== undefined && reasoningEfforts.length === 0 ? undefined : reasoningEfforts; +} + /** Enumerate the current ordinary Combo eligibility set without retaining attempted picks. */ function eligibleJevComboChoices( config: OcxConfig, @@ -269,23 +339,8 @@ function eligibleJevComboChoices( excluded.add(key); // The TypeSafe row owns a decision credential, not an inference transport. if (pick.target.provider === JEV_PROVIDER_ID) continue; - let ladder: string[] | undefined; - try { - const route = routeConcreteModel(config, key); - ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId }); - } catch { - // Preserve the existing routing-failure surface. Unknown capability becomes the explicit - // no-effort choice rather than broadening JEV's effort allowlist. - ladder = undefined; - } - const supportedEfforts = (ladder ?? []).filter(isCodexReasoningEffort) as OcxComboDefaultEffort[]; - const configuredEfforts = pick.target.reasoningEfforts; - const reasoningEfforts = configuredEfforts === undefined - ? supportedEfforts - : configuredEfforts.filter(effort => supportedEfforts.includes(effort)); - // An explicit allowlist is restrictive. If catalog capabilities drift until no configured - // effort remains supported, omit the target instead of silently broadening JEV's choices. - if (configuredEfforts !== undefined && reasoningEfforts.length === 0) continue; + const reasoningEfforts = jevChoiceEfforts(config, pick.target); + if (!reasoningEfforts) continue; choices.push({ pick: { ...pick, attempted: [key] }, candidate: { @@ -549,7 +604,8 @@ export async function executeComboResponses( ); return options.abortSignal?.aborted ? clientCancelledResponse() - : comboUnavailable(comboId); + : comboUnavailable(comboId, undefined, + comboExclusionReasons(config, comboId, target => meteredAllowed || !target.metered)); } if (!(await recoverUnreadableEncryptedTask())) { return options.abortSignal?.aborted @@ -565,13 +621,16 @@ export async function executeComboResponses( if (onlyReplayIncompatibleTargetsRemain()) return targetIncompatibleResponse(); return options.abortSignal?.aborted ? clientCancelledResponse() - : comboUnavailable(comboId); + : comboUnavailable(comboId, undefined, comboExclusionReasons(config, comboId, targetEligible)); } let jevDecision: JevDecision | undefined; if (combo.strategy === "jev") { const choices = eligibleJevComboChoices(config, comboId, targetEligible, Date.now()); const first = choices[0]; - if (!first) return comboUnavailable(comboId); + if (!first) { + return comboUnavailable(comboId, undefined, comboExclusionReasons(config, comboId, target => + targetEligible(target) && target.provider !== JEV_PROVIDER_ID && jevChoiceEfforts(config, target) !== undefined)); + } const resolvedFailOpenEffort = resolveEffortAtOrBelow( "medium", first.candidate.reasoningEfforts, @@ -685,6 +744,9 @@ export async function executeComboResponses( // is gone, so carry the loop's own classification decision instead of re-deriving a // weaker one from the status alone (#4149). let lastFailureClassifiesOverflow = false; + // Local input admission refusals (#1524), while every failed target so far was one: such a + // target sent nothing, so an exhausted ladder must not be reported as a provider rejection. + let localAdmissionRefusals: string[] | undefined = []; // The first quota/rate-limit refusal of the ladder. A fallback that then refuses its own // credential or plan (401/403, or a 400 the ladder hopped past) says nothing about why the // request failed: the primary target ran out. Returning that fallback refusal told clients @@ -1052,6 +1114,12 @@ export async function executeComboResponses( Date.now() - started, failure.usage, ); + if (failure.upstreamCode === "input_admission_refused" && attempt.sendCount === 0) { + attempt.errorCode = failure.upstreamCode; + localAdmissionRefusals?.push(failure.upstreamMessage ?? ""); + } else { + localAdmissionRefusals = undefined; + } attemptRetained = true; lastFailure = failure.response; if (!quotaFailure && (lastFailure.status === 429 || lastFailure.status === 402)) { @@ -1135,12 +1203,16 @@ export async function executeComboResponses( const failedTargetKey = targetKey(pick.target); let failedTargetCooldownRecorded = false; // The dispatch rewrote this to name the pool account that actually served the turn. - const failedAccount = cooledPoolAccountLabel( - config, - pick.target.provider, - pick.target.model, - poolAccountProviderLabel(childLog.provider, pick.target.provider), - ); + const poolAccountLabel = poolAccountProviderLabel(childLog.provider, pick.target.provider); + const failedAccount = failure.response.status === 402 + ? creditRefusedAnthropicAccountLabel( + config, pick.target.provider, pick.target.model, poolAccountLabel, failure.classificationText, failure.upstreamCode, + failure.response, + ) + : cooledPoolAccountLabel(config, pick.target.provider, pick.target.model, poolAccountLabel) + ?? (failure.response.status === 429 && failureCooldownScope === "target" + ? cooledGenericOAuthAccountLabel(config, pick.target.provider, pick.target.model, childLog.accountLogLabel, failure.retryAfter) + : undefined); // The target still cools on the pool's own refusal (the anti-hammer guard), but for the local // fallback rather than the pool's restated per-account Retry-After. const poolLocalRefusal = isAnthropicPoolLocalRefusal( @@ -1232,9 +1304,14 @@ export async function executeComboResponses( lastFailure?.status === 413 && lastFailureClassifiesOverflow ) { + if (localAdmissionRefusals?.length) logCtx.errorCode = "input_admission_refused"; + const localMessage = localAdmissionRefusals?.length + ? `opencodex refused this turn locally; nothing was sent upstream. No target of combo ${comboId} ` + + `that was tried can fit its input: ${localAdmissionRefusals.join(" ")} Compact the conversation before retrying.` + : undefined; return (rawBody as { stream?: unknown } | null)?.stream === true - ? streamingContextOverflowResponse(requestedModel, options.translatorBudget) - : jsonContextOverflowResponse(); + ? streamingContextOverflowResponse(requestedModel, options.translatorBudget, localMessage) + : jsonContextOverflowResponse(localMessage); } return failure; } diff --git a/src/server/responses/core-errors.ts b/src/server/responses/core-errors.ts index 8e067cad317..32fa78e6ef9 100644 --- a/src/server/responses/core-errors.ts +++ b/src/server/responses/core-errors.ts @@ -8,7 +8,7 @@ import { DecompressedBodyTooLargeError, describeInboundBodyRefusal, } from "../request-decompress"; -import { comboCooldownRetryAfterSeconds } from "../../combos"; +import { comboCooldownRetryAfterSeconds, NoAvailableComboTargetsError } from "../../combos"; import type { AgentTaskRecoveryFailureReason } from "./agent-task-recovery"; /** @@ -115,8 +115,8 @@ export function comboUnavailableResponse( } -export function comboUnavailable(comboId: string, now = Date.now()): Response { - return comboUnavailableResponse(`No available targets for combo: ${comboId}`, { +export function comboUnavailable(comboId: string, now = Date.now(), reasons = ""): Response { + return comboUnavailableResponse(new NoAvailableComboTargetsError(comboId, reasons).message, { retryAfter: comboCooldownRetryAfterSeconds(comboId, now), }); } diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 35871ccfc70..3b7010d4d2f 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -17,7 +17,7 @@ import { unreadableEncryptedAgentTaskResponse, } from "./core-errors"; import { parseSyntheticRowId } from "../fast-row"; -import { resolveComboId, comboIdFromRawBody, NoAvailableComboTargetsError, pickComboTarget } from "../../combos"; +import { resolveComboId, comboIdFromRawBody, comboExclusionReasons, NoAvailableComboTargetsError, pickComboTarget } from "../../combos"; import { INTERCEPT_TARGET_UNAVAILABLE_CODE, interceptTargetUnavailableResponse, resolveChosenTarget, resolveShadowCallTarget } from "./shadow-target-availability"; import { MEMORY_MODEL_TARGET_UNAVAILABLE_CODE, @@ -645,7 +645,10 @@ export async function prepareResponsesRequest( config, options.admission, config.combos?.[candidate.combo.comboId], )) { const pick = pickComboTarget(config, candidate.combo.comboId, { eligible: target => !target.metered }); - if (!pick) throw new NoAvailableComboTargetsError(candidate.combo.comboId); + if (!pick) { + throw new NoAvailableComboTargetsError(candidate.combo.comboId, + comboExclusionReasons(config, candidate.combo.comboId, target => !target.metered)); + } const routed = routeConcreteModel(config, `${pick.target.provider}/${pick.target.model}`); candidate = { ...routed, combo: pick, routeKind: "combo", routeReason: routed.routeReason === "blocked-model-redirect" ? "blocked-model-redirect" : "combo-pick" }; } @@ -788,7 +791,7 @@ export async function prepareResponsesRequest( if (policyRefusal) return policyRefusal; if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err); if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailable(err.comboId); + return comboUnavailable(err.comboId, undefined, err.reasons); } if (err instanceof NoEligiblePolicyCandidateError) { // Persist the evaluation trace (per-candidate exclusions + the @@ -1006,7 +1009,7 @@ export async function prepareResponsesRequest( if (policyRefusal) return policyRefusal; if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err); if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailable(err.comboId); + return comboUnavailable(err.comboId, undefined, err.reasons); } if (err instanceof NoEligiblePolicyCandidateError) { logCtx.routeDecision = err.trace; @@ -1223,7 +1226,7 @@ export async function prepareResponsesRequest( if (policyRefusal) return policyRefusal; if (err instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(err); if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailable(err.comboId); + return comboUnavailable(err.comboId, undefined, err.reasons); } if (err instanceof NoEligiblePolicyCandidateError) { logCtx.routeDecision = err.trace; diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index ab6ba0e6dd4..1e5130316ad 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -418,7 +418,8 @@ export async function executeResponsesSidecars( hop.permit?.release(); return null; } - recoveryKind = originalResponse?.status === 401 ? "oauth-401" : "anthropic-oauth-429"; + recoveryKind = originalResponse?.status === 401 ? "oauth-401" + : originalResponse?.status === 403 ? "oauth-account-403" : "anthropic-oauth-429"; sidecarBudget.ownCredentialHop(hop.permit); } else { // No key pool, no generic OAuth roster, no Anthropic pool could produce a replacement diff --git a/structure/dashboard-and-usage.md b/structure/dashboard-and-usage.md index 8254b5a1de5..c246b75a64f 100644 --- a/structure/dashboard-and-usage.md +++ b/structure/dashboard-and-usage.md @@ -279,7 +279,7 @@ inheriting its bounds, so deleting a ledger row removes it from the grouping. `usageLedgerMaxBytes` is unset by default; when set, an append that crosses it publishes the newest whole rows byte for byte through the shared atomic writer, refuses the rename unless the source is the exact revision that was copied, and then discards the Logs ring, the retained aggregates and the -request-history index so no surface serves rows the ledger no longer has. That index (`src/routing/history/`) rebuilds a ledger of at least `REQUEST_HISTORY_WORKER_REBUILD_MIN_BYTES` (1 MiB) on a worker thread (#6990): `rebuild-worker.ts`, driven by `rebuild-job.ts`, ingests the ledger snapshot the refresh recorded into a private `.rebuild--` file through the same `ingest.ts` the inline path uses, switches it to WAL and closes it; the serving thread then closes the old index and renames the build over it. Async readers wait for the build; the routing-time sync entry never does and throws `RequestHistoryIndexBusyError`, which health scoring reads as unknown evidence. A close discards the build, rejects waiting readers with `RequestHistoryIndexClosedError`, and removes the build files only after the worker thread exits. For 30 s after a worker build fails (other than on a replaced ledger), async readers and manual rebuilds rebuild inline instead, while routing-time reads keep reporting busy. +request-history index so no surface serves rows the ledger no longer has. That index (`src/routing/history/`) rebuilds a ledger of at least `REQUEST_HISTORY_WORKER_REBUILD_MIN_BYTES` (1 MiB) on a worker thread (#6990): `rebuild-worker.ts`, driven by `rebuild-job.ts`, ingests the ledger snapshot the refresh recorded into a private `.rebuild--` file through the same `ingest.ts` the inline path uses, switches it to WAL and closes it; the serving thread then closes the old index and renames the build over it. Async readers wait for the build; the routing-time sync entry never does and throws `RequestHistoryIndexBusyError`, which health scoring reads as unknown evidence. Health scoring (`src/routing/health.ts`) counts physical sends only: a request answered locally or refused before any send is no evidence for any target, and a row's terminal status and close reason score its last physical send, so a 200 stream cut mid-flight after a failover counts against the target that dropped it (`tests/server/health-scoring.test.ts`). A close discards the build, rejects waiting readers with `RequestHistoryIndexClosedError`, and removes the build files only after the worker thread exits. For 30 s after a worker build fails (other than on a replaced ledger), async readers and manual rebuilds rebuild inline instead, while routing-time reads keep reporting busy. Ledger revisions in `src/usage/log.ts` capture device/inode identifiers from bigint descriptor stats as exact decimal strings; history metadata, ingestion, cooperative scans and retention compare them without narrowing to Numbers, preserving distinct NTFS identifiers above 2^53. An older index containing a rounded identifier fails the identity check and rebuilds once. `src/usage/summary.ts` turns that file into the `/api/usage` shape — totals, daily zero-filled grid, model and provider breakdowns, and `measured / reported / unreported / unsupported / estimated` counts. Each scope also aggregates end-to-end output throughput (#6309): measured output tokens and wall-clock `durationMs` are summed over attempts reporting both, exposing `throughputTokensPerSec` — a token-sum-over-duration-sum, never a mean of per-request rates. `throughputSamples` counts qualifying attempts (legacy rows without attempts contribute one); nonpositive or nonfinite tokens/timing are excluded. The Usage summary and Models/Providers tables display the rate and sample basis, or unavailable when none qualify. All three throughput displays use the same finite-number guard as their tooltip; malformed or nonfinite rates are unavailable rather than formatted. diff --git a/structure/providers/anthropic-account-pool.md b/structure/providers/anthropic-account-pool.md index 34cbe3d10cb..1981b3a9261 100644 --- a/structure/providers/anthropic-account-pool.md +++ b/structure/providers/anthropic-account-pool.md @@ -300,11 +300,16 @@ only canonical Fable, Opus, or Sonnet labels after removing terminal controls; u `src/oauth/anthropic-account-refusal.ts` accepts an HTTP 403 only when a complete bounded JSON `error` envelope has `permission_error` or `billing_error` and a whole-message -account subscription/Claude Code entitlement or Anthropic credit-balance refusal. +account subscription/Claude Code entitlement or Anthropic credit-balance refusal, or when +the envelope carries the structured `oauth_org_not_allowed` type or code (organization disabled +Claude subscription access; also matched by its whole message under `permission_error`). Generic permission, resource/model access, content policy, quoted diagnostics, conflicting error codes and incomplete/malformed bodies remain terminal. The shared physical dispatch in `src/server/responses/request-transport.ts` binds the refusal to the stored bearer it sent; overridden headers, additional API keys and replaced credentials cannot cool that account. +Combo 402 credit-balance refusals carry that immutable send proof across bounded error +normalization in `src/server/responses/core-combo-failure.ts`. The combo rechecks instance, +account incarnation, credential generation and pause/reauth state before recording entitlement cooldown. `src/oauth/anthropic-routing.ts` records a process-local cooldown, clears account affinity, and applies the existing strategy, pause/reauth exclusions, model route and selection commit. @@ -313,6 +318,8 @@ A usage probe cannot clear this non-reset-derived cooldown. Plan renewal therefo no permanent reauthentication flag, though a renewed account waits for cooldown expiry. The main dispatch, pre-output empty-completion retry, web-search and image bridge use the same bounded recovery and record the final refusal even when no retry sends remain. +Each records the rotated send as `oauth-account-403` (cause `credential-rejected`), not the +quota recovery `anthropic-oauth-429`, as the native Messages path already did. A streamed terminal continuation after assistant output and search after live output or a published search call cannot rotate on 403. Non-streaming buffered continuations may recover. No eligible replacement preserves the upstream 403; generic 401 handling is unchanged. diff --git a/structure/runtime.md b/structure/runtime.md index dc720c31f82..1cf4edf64ed 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -504,7 +504,7 @@ This is also why the classifier cannot duplicate visible output. Native byte str Regression coverage: `tests/responses/responses-forward-prompt-envelope.test.ts`, `tests/routing/router-combo-failover-classification.test.ts`, `tests/routing/routing-policy-fallback.test.ts`, `tests/helpers/combo-context-overflow-cases.ts`, and `tests/server/server-combo-failover-e2e.test.ts`. -`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; unknown accounts, ambiguous labels, API-key and generic OAuth providers, and other failures keep their ordinary target cooldown. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. +`src/combos/failover.ts` uses a 10-minute fallback for a spent account usage window (codes `usage_limit_exceeded`, `usage_limit_reached`, `1308`, or `usage limit reached` / `usage limit has been reached` / `token-plan quota has been exhausted` prose, including HTTP 502) and for provider-scoped credential or billing failure codes such as `invalid_api_key` and `insufficient_quota`. This duration does not change failure classification or cooldown scope; upstream retry/reset signals and configured durations retain precedence. It caps explicit upstream `Retry-After` target cooldowns at 24 hours while reset-derived, configured, and fallback cooldowns remain capped at 10 minutes. `src/combos/resolve.ts` omits a target cooldown for a 429 only when the failed child identifies one account in an Anthropic OAuth or canonical Codex pool and that account has a live pool cooldown, or identifies one account in a generic OAuth pool that still has an eligible sibling. `src/server/responses/core-combo.ts` checks the pool's health record before passing that attribution; for a generic OAuth pool it first hands the account (matched through its `accountLogLabel`) to `rotateGenericOAuthAccountOn429`, which cools it under the pool's own Retry-After policy even when the per-request account budget ended rotation before that account was recorded, and attributes the failure only when the rotator names a sibling. Unknown accounts, ambiguous labels, a lone or fully cooled generic OAuth pool, API-key providers, and other failures keep their ordinary target cooldown. An API-key pool needs no attribution: its 429 rotation walks every key inside the attempt, so a 429 reaches the combo only once every key is cooled. Regression coverage: `tests/oauth/generic-oauth-combo-account-cooldown.test.ts`. A 402 is the provider-scoped twin for an Anthropic OAuth pool: the pool rotates only on 401/403/429, so when the failed child identifies one pool account and the failure carries the exact credit-balance refusal (`isAnthropicBillingFailure`), `core-combo.ts` records that account's refusal through the pool's `recordAnthropicAccountRefusal` (the same entitlement backoff a 403 credit refusal gets) and `resolve.ts` omits every target cooldown only while the pool still has an eligible account; a lone account, an exhausted pool, an unidentified account, or any other 402 keeps the provider-scoped cooldown. Regression coverage: `tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts`. The Anthropic OAuth pool's own all-accounts-cooled 429 is the one unattributed exception: `isAnthropicPoolLocalRefusal` in `core-combo.ts` requires the `anthropic` provider with `authMode: "oauth"`, status 429, no identified failed account, no eligible pool account and a live pool retry-after. The target then still cools, but for the local fallback instead of the restated per-account `Retry-After`, so a resumed or newly added account is not ignored until a 24-hour target cooldown expires. An upstream 429 on an API-key provider or on an identified pool account keeps its full `Retry-After`. Regression coverage: `tests/routing/always-on-429-failover.test.ts`. Cooldown entries in `src/combos/failover.ts` retain the failure status. The quota-only snapshot query records target/expiry pairs for unexpired 429/402 cooldowns using only cooldown map lookups. Its config-aware wrapper in `src/combos/resolve.ts` applies the picker's provider availability and cached quota checks before dispatch; request eligibility runs lazily only when exhaustion considers a 503, with throwing checks treated as ineligible. Responses exhaustion uses this evidence under the [failover contract](transports/responses-failover.md); ordinary cooldown selection and timing stay unchanged. A zero-attempt `combo_unavailable` names why each target was skipped. `comboExclusionReasons` in `src/combos/resolve.ts` derives one coarse label per target from the picker's own checks, in picker order: `provider missing`, `provider disabled`, `quota exhausted`, `cooldown s`, then `ineligible for this request` when the caller's eligibility predicate refuses it or throws. `NoAvailableComboTargetsError` and `comboUnavailable` append them to the message; the label carries only the configured provider/model, never account, credential or request data. Coverage: `tests/responses/responses-combo-exhausted-error.test.ts`. ## Combo default effort precedence diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index b77a49917c4..369ea8947d7 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -1,7 +1,7 @@ # Responses Failover And Replay Combo `metered` eligibility follows the [runtime consent contract](../runtime.md#request-local-target-compatibility), including recovery selections. In `src/server/responses/run-turn-execution.ts`, streaming and buffered pre-output failures retain structured adapter status/type/code via `adapterFailureFromEvent`, and `formatErrorResponse`'s `classified` option keeps that verdict in the child body instead of re-deriving it from the status, wherever its type and message add nothing to what the status alone says; a verdict read from the message (quota, credentials, cyber policy, context overflow, catalog size) still outranks a generic adapter code, and spend-ledger, cyber-policy and replay-refusal overrides still win, and `consumeComboFailure` re-applies the client allowlist. Spend-ledger storage refusal remains 503. This preserves an exhausted subscription chain’s 429 response, a child's `insufficient_quota`, and the cool-free hop on `unsupported_input_modality` in `src/combos/failover.ts`. Coverage: `tests/adapters/anthropic/anthropic-subscription-chain.test.ts` and `tests/responses/responses-combo-run-turn-classified-failure.test.ts`. `src/server/responses/request-transport.ts` resolves the final Anthropic model ID once for each enabled-pool request and holds its model route through admission. The three account-refusal retry sites in `adapter-dispatch.ts`, `adapter-continuation.ts` and `sidecar-execution.ts` use the same route, keep the original refusal if no replacement exists inside it, and preserve existing send and output replay limits. A local cooldown returns 429 with the earliest known Retry-After among accounts the route can use when they recover, and answers a usage limit when every held account is on spent quota ([quota exhaustion answer](../providers/anthropic-account-pool.md#quota-exhaustion-answer-and-fresh-read-recovery)); explicit fallback includes usable ordinary-pool accounts even when the saved route IDs have been removed, while a strict route stays route-scoped. Selection, refusal and 429-rotation logs use the rule’s 1-based `route:#` position, never its name. -Classified 429 admission follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#classified-429-admission). Classified account-entitlement 403 recovery follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery). +Classified 429 admission follows the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#classified-429-admission). Classified account-entitlement 403 recovery and combo 402 credit-refusal cooldowns follow the [Anthropic account-pool contract](../providers/anthropic-account-pool.md#account-entitlement-refusal-recovery); combo error normalization preserves the original physical-send proof, and a stale credential cannot cool its replacement. `src/server/responses/compaction-recovery-policy.ts` is a pure eligibility policy, not a dispatcher. It requires explicit configuration and normalized attempt evidence, preserves ordinary requests, and refuses cancellation, committed semantic output, tool effects, protected failures, exhausted @@ -508,7 +508,7 @@ existing combo policy already treats as a safe hop. That ordering is the whole p total window cannot hold the turn plus the caller's allowance answers 200, emits a few hundred tokens and stops on `finish_reason: length`, which the Anthropic surface renders as an output-token error naming a limit the model never approached — and by then output has committed and no later -target may be tried. +target may be tried. When every failed target of an exhausted combo was refused this way with a send count of 0, `src/server/responses/core-combo.ts` keeps the `context_length_exceeded` overflow envelope that clients compact on, but replaces the provider wording from `context-overflow.ts` with a proxy-owned message saying the turn was refused locally, carrying each tried target's estimate and window (a target skipped for cooldown is not claimed to be too small). Those attempts, and the final row, keep the code `input_admission_refused` instead of `http_413`. One target that reached a provider keeps the provider wording for the whole combo. Scope is deliberately narrow. Direct and single-target requests keep the loose 2.5x pathological-input gate, because they have nowhere to hop. Compaction turns stay exempt. Unknown @@ -517,8 +517,8 @@ limits for custom providers. Canonical native slugs that the narrower pinned tab resolve their window from the generated in-tree bundle, which is what made the gate inert on the route where this was first observed; explicit provider and operator caps may only narrow it. -Regression coverage: `tests/server/input-admission.test.ts` and -`tests/helpers/combo-context-headroom-cases.ts`. +Regression coverage: `tests/server/input-admission.test.ts`, +`tests/helpers/combo-context-headroom-cases.ts` and `tests/responses/responses-context-overflow.test.ts`. Native steering retains fixed phase deadlines and reconciled replay output; see the [steering stability contract](../transports/streaming-health.md#steering-deadlines-and-replay-completeness). diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 1caf9c4f093..b08fc1a3593 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -434,7 +434,7 @@ is composed from the following owners in `src/server/responses/`; none is a gene Reusable helpers live in `core-auth.ts`, `core-codex-account.ts`, `core-combo.ts`, `core-combo-failure.ts`, `core-combo-native.ts`, `combo-requested-effort.ts`, `core-errors.ts`, `core-lifetime.ts`, `core-normalize.ts`, `core-opaque-recovery.ts` and `core-replay.ts`. `core-options.ts` owns the public option types -and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. +and small composition contracts, including [finite model-refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence). Existing public helper names are re-exported by `core.ts`. After an admitted Anthropic OAuth pool rotation, `adapter-dispatch.ts` rebinds reasoning replay to the committed credential snapshot; `adapter-continuation.ts` rebinds both its continuation clone and the outer request that persists the response. The replacement send strips the previous account's private state, while persisted ownership and the next turn follow the serving account. Codex alternate-account sends in `core-codex-account.ts` record `oauth-account-429` for quota recovery and `transient-5xx` for transient recovery, preserving the first response's wire status when sealing its attempt; gated-model moves keep their existing unattributed kind. Adapter construction remains with the existing registry; `fetch-helpers.ts` remains a leaf. For Kiro OAuth with load settings, `request-transport.ts` acquires a lease on the admitted account and transfers it before a reactive replacement send; cancellation permanently fences the request holder so recovery cannot install a late lease after abort cleanup. `core.ts` and `core-lifetime.ts` release the lease on returned-body completion, error, or cancellation, outside the inner admission `finally`. Mutable values are not copied across phases. A phase exposes only the values consumed by later diff --git a/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts index 79d589407c4..559ade3e2e1 100644 --- a/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts +++ b/tests/adapters/anthropic/anthropic-combo-account-cooldown.test.ts @@ -182,3 +182,76 @@ test("a lone OAuth account without pool cooldown still cools the combo target", expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); expect(sentTokens).toHaveLength(1); }); + +/** The pool's own credit-balance wording (`isAccountRefusal`), on a 402. */ +function creditRefused(): Response { + return Response.json({ type: "error", error: { + type: "billing_error", + message: "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", + } }, { status: 402 }); +} + +test("one account's 402 credit refusal does not cool the combo target while a sibling is eligible", async () => { + const ids = await seed(2); + const config = configFor(() => answered()); + const anthropic = config.providers.anthropic as OcxProviderConfig & { fetch: typeof fetch }; + // Only the first account is out of credit; the second still answers. + anthropic.fetch = (async (_input, init) => { + const token = String(new Headers(init?.headers).get("authorization")); + sentTokens.push(token); + return token === `Bearer ${credential(0).access}` ? creditRefused() : answered(); + }) as typeof fetch; + const fallbackSends: string[] = []; + config.providers.fallback = { + adapter: "anthropic", baseUrl: "https://fallback.test", authMode: "key", apiKey: "synthetic-fallback-key", + models: [MODEL], fetch: (async () => { fallbackSends.push("B"); return answered(); }) as typeof fetch, + } as OcxProviderConfig; + config.combos = { pooled: { strategy: "failover", targets: [TARGET, { provider: "fallback", model: MODEL }] } }; + + expect((await post(config)).status).toBe(200); + expect(sentTokens).toEqual([`Bearer ${credential(0).access}`]); + expect(fallbackSends).toEqual(["B"]); + // The refusal names one account: the pool cools it, and the target stays usable for the other. + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + expect(getAnthropicAccountHealthSnapshot(ids[0]!)).not.toBeNull(); + + sentTokens = []; + expect((await post(config)).status).toBe(200); + expect(sentTokens).toEqual([`Bearer ${credential(1).access}`]); + expect(fallbackSends).toEqual(["B"]); +}, 20_000); + +test("402 credit refusals from every pool account still cool the combo target", async () => { + await seed(2); + const config = configFor(creditRefused); + // The first refusal leaves a sibling and spares the target; the second leaves none. + for (let attempt = 0; attempt < 2; attempt++) expect((await post(config)).status).toBe(402); + expect(sentTokens).toHaveLength(2); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); +}, 20_000); + +test("a 402 consumed after re-login cannot cool the replacement credential in the same account slot", async () => { + const ids = await seed(2); + const replacement = { ...credential(2), accountId: credential(0).accountId }; + let replaced = false; + const config = configFor(() => new Response(new ReadableStream({ + async pull(controller) { + // Zero buffering starts this only when recovery reads the physically bound response body. + await saveCredential("anthropic", replacement); + replaced = true; + controller.enqueue(new TextEncoder().encode(JSON.stringify({ type: "error", error: { + type: "billing_error", + message: "Your credit balance is too low to access the Anthropic API.", + } }))); + controller.close(); + }, + }, { highWaterMark: 0 }), { status: 402, headers: { "content-type": "application/json" } })); + + expect((await post(config)).status).toBe(402); + expect(replaced).toBe(true); + expect(sentTokens).toEqual([`Bearer ${credential(0).access}`]); + const current = getAccountSet("anthropic")!.accounts.find(account => account.id === ids[0]); + expect(current?.credential.access).toBe(replacement.access); + expect(getAnthropicAccountHealthSnapshot(ids[0]!)).toBeNull(); + expect(getAnthropicAccountHealthSnapshot(ids[1]!)).toBeNull(); +}, 20_000); diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 1d1bfab31e1..c07ade55381 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -10,6 +10,7 @@ import { getAccountSet, saveAccountCredential, saveCredential, setActiveAccount import { clearAccountQuotaCache, getCachedProviderAccountQuota, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; import { clearResponseStateForTests } from "../../../src/responses/state"; import { handleResponses } from "../../../src/server/responses"; +import type { RequestLogContext } from "../../../src/server/request-log"; import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; @@ -156,11 +157,11 @@ function configFor(reply: (body: Record) => Response | Promise< }; } -function post(config: OcxConfig, body: Record = {}) { +function post(config: OcxConfig, body: Record = {}, logCtx: RequestLogContext = { model: "", provider: "" }) { return handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false, ...body }), - }), config, { model: "", provider: "" }); + }), config, logCtx); } function expectQuota(id: string, fiveHourPercent: number, weeklyPercent: number) { @@ -406,7 +407,25 @@ test.each([ expect(sent).toHaveLength(2); }); +const orgDisabled = "Your organization has disabled Claude subscription access for Claude Code. Use an Anthropic API key instead, or ask your admin to enable access."; +test.each([ + { type: "permission_error", message: orgDisabled }, + { type: "oauth_org_not_allowed", message: orgDisabled }, + { type: "permission_error", code: "oauth_org_not_allowed", message: "Access denied." }, +])("organization-disabled 403 rotates and cools the refused account (%#)", async error => { + const [a, b] = await seed(); + const config = configFor(body => sent.length === 1 + ? Response.json({ type: "error", error }, { status: 403 }) : answer(body.stream === true)); + const response = await post(config); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expect(getEligibleAnthropicAccounts()).toEqual([b!]); + expect(getAnthropicAccountHealthSnapshot(a!)).not.toBeNull(); +}); + test.each([ + [`Request denied: ${orgDisabled}`, "permission_error"], ["Request blocked by content policy", "permission_error"], ["You do not have permission to access this resource", "permission_error"], ["Your account does not have access to this model", "permission_error"], @@ -447,6 +466,28 @@ test.each(["search", "image"])("real %s sidecar recovers account 403", async kin expect(getAnthropicAccountHealthSnapshot(a!)).not.toBeNull(); }); +// An account refusal is a credential denial, not a quota spend: it must not count as a rate limit. +test.each(["main", "search", "image", "continuation"])("%s account 403 rotation records oauth-account-403", async path => { + await seed(); + const refusalAt = path === "continuation" ? 2 : 1; + const config = configFor(body => sent.length === refusalAt ? accountRefusal() + : answer(body.stream === true, "0.1", "0.2", path === "continuation" && sent.length === 1 ? "" : "The answer is complete.")); + if (path === "search") config.webSearchSidecar = { backend: "anthropic", enabled: true }; + if (path === "image") { + config.images = { bridgeEnabled: true }; + config.providers.xai = { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", apiKey: "synthetic-image-key" }; + } + if (path === "continuation") config.emptyCompletionRetry = true; + const tools = path === "search" ? [{ type: "web_search" }] : path === "image" ? [{ type: "image_generation" }] : undefined; + const logCtx: RequestLogContext = { model: "", provider: "" }; + const response = await post(config, { stream: true, ...(tools ? { tools } : {}) }, logCtx); + expect(await response.text()).toContain("The answer is complete."); + expect(sent).toHaveLength(refusalAt + 1); + const kinds = [...(logCtx.attempts ?? []), logCtx.activeAttempt].flatMap(attempt => attempt?.recoveryKinds ?? []); + expect(kinds).toContain("oauth-account-403"); + expect(kinds).not.toContain("anthropic-oauth-429"); +}); + test("empty pre-output continuation recovers account 403", async () => { await seed(); const config = configFor(body => { diff --git a/tests/adapters/terminal-continuation-owner-rotation.test.ts b/tests/adapters/terminal-continuation-owner-rotation.test.ts index 69ce097aca4..74673b1891a 100644 --- a/tests/adapters/terminal-continuation-owner-rotation.test.ts +++ b/tests/adapters/terminal-continuation-owner-rotation.test.ts @@ -5,7 +5,10 @@ import { join } from "node:path"; import type { ProviderAdapter } from "../../src/adapters/base"; import { saveConfig } from "../../src/config"; import { clearKeyCooldowns } from "../../src/providers/key-failover"; -import { reasoningReplayKeyCredentialIdentity } from "../../src/responses/reasoning-replay-cache"; +import { clearAnthropicAccountPoolState } from "../../src/oauth/anthropic-routing"; +import { getValidAccessSnapshotForAccount } from "../../src/oauth"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { reasoningReplayKeyCredentialIdentity, reasoningReplayOAuthCredentialIdentity } from "../../src/responses/reasoning-replay-cache"; import { clearResponseStateForTests, previousResponseProviderState, @@ -22,6 +25,7 @@ import { removeTreeWithRetry } from "../helpers/remove-tree"; interface BuildObservation { key: string; continuation?: string; + stripReasoning?: true; } let builds: BuildObservation[] = []; @@ -76,7 +80,8 @@ const actualResolveAdapter = actualResolver.resolveAdapter; mock.module("../../src/server/adapter-resolve", () => ({ ...actualResolver, resolveAdapter(provider: OcxProviderConfig, cacheRetention?: "none" | "short" | "long") { - if (provider.adapter !== "test-terminal-owned") { + // Match on the destination: the "anthropic" provider is wire-pinned to the anthropic adapter. + if (provider.baseUrl !== "https://owned-terminal.test/v1") { return actualResolveAdapter(provider, cacheRetention); } const key = provider.apiKey ?? ""; @@ -86,7 +91,11 @@ mock.module("../../src/server/adapter-resolve", () => ({ name: "anthropic", buildRequest(parsed: OcxParsedRequest) { const continuation = parsed._providerContinuation?.kiro?.conversationId; - builds.push({ key, ...(continuation ? { continuation } : {}) }); + builds.push({ + key, + ...(continuation ? { continuation } : {}), + ...(parsed._stripReasoningEncryptedContent ? { stripReasoning: true as const } : {}), + }); return { url: "https://owned-terminal.test/v1/messages", method: "POST", @@ -121,6 +130,7 @@ describe("terminal continuation provider-owner rotation", () => { // Take the writer lease after this case installs its home so direct handler dispatch can open the spend journal. releaseSpendHome = acquireOwnedSpendHome(); builds = []; + clearAnthropicAccountPoolState(); clearKeyCooldowns(); clearResponseStateForTests(); }); @@ -130,6 +140,7 @@ describe("terminal continuation provider-owner rotation", () => { releaseSpendHome?.(); releaseSpendHome = undefined; globalThis.fetch = originalFetch; + clearAnthropicAccountPoolState(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; clearKeyCooldowns(); @@ -250,4 +261,93 @@ describe("terminal continuation provider-owner rotation", () => { ]); expect(phases).toEqual([]); }); + + test.each([ + ["initial send", false], + ["terminal continuation", true], + ] as const)("Anthropic pool rotation on the %s rebinds the replay owner to the rotated account", async (_label, continuation) => { + for (let index = 0; index < 2; index += 1) { + await saveCredential("anthropic", { + access: `anthropic-access-${index}`, + refresh: `anthropic-refresh-${index}`, + expires: Date.now() + 3_600_000, + accountId: `anthropic-account-${index}`, + }, { addAccount: true }); + } + const [idB, idA] = getAccountSet("anthropic")?.accounts.map(account => account.id) ?? []; + expect(await setActiveAccount("anthropic", idA!)).toBe(true); + const ownerA = reasoningReplayOAuthCredentialIdentity(await getValidAccessSnapshotForAccount("anthropic", idA!)); + const ownerB = reasoningReplayOAuthCredentialIdentity(await getValidAccessSnapshotForAccount("anthropic", idB!)); + expect(ownerB).toBeDefined(); + expect(ownerB).not.toBe(ownerA); + const config: OcxConfig = { + port: 0, + defaultProvider: "anthropic", + providers: { + anthropic: { + adapter: "test-terminal-owned", + baseUrl: "https://owned-terminal.test/v1", + authMode: "oauth", + models: ["model"], + }, + }, + } as OcxConfig; + saveConfig(config); + + const phases = ["seed", ...(continuation ? ["plan"] : []), "rate-limit", "rotated", "follow"]; + const seenAuthorization: string[] = []; + globalThis.fetch = (async (_input, init) => { + seenAuthorization.push(new Headers(init?.headers).get("authorization") ?? ""); + const phase = phases.shift(); + if (!phase) throw new Error("unexpected extra upstream request"); + if (phase === "rate-limit") { + return Response.json({ error: { message: "rotate" } }, { status: 429, headers: { "retry-after": "30" } }); + } + return new Response("", { headers: { "x-test-phase": phase } }); + }) as typeof fetch; + // A stable thread binds the reasoning-replay serving identity across turns. + const post = (body: Record) => handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", "thread-id": `thread-anthropic-rotation-${continuation}` }, + body: JSON.stringify({ model: "anthropic/model", stream: false, store: true, ...body }), + }), + config, + { model: "", provider: "" }, + ); + + const seed = await post({ input: "seed" }); + expect(seed.status).toBe(200); + const seedJson = await seed.json() as { id: string }; + expect(previousResponseProviderState(seedJson.id)).toMatchObject({ __ocxOwner: { credentialIdentity: ownerA } }); + + const rotated = await post({ + previous_response_id: seedJson.id, + input: continuation ? "Please modify the file now" : "hello", + tools: [{ type: "function", name: "read_file", description: "read a file", parameters: { type: "object" } }], + }); + expect(rotated.status).toBe(200); + const rotatedJson = await rotated.json() as { id: string }; + // The turn was served by B, so B owns the state persisted for the next turn. + expect(previousResponseProviderState(rotatedJson.id)).toMatchObject({ + __ocxOwner: { credentialIdentity: ownerB }, + kiro: { conversationId: "private-b" }, + }); + + const follow = await post({ previous_response_id: rotatedJson.id, input: "follow up" }); + expect(follow.status).toBe(200); + const a = "anthropic-access-1"; + const b = "anthropic-access-0"; + expect(seenAuthorization).toEqual([a, ...(continuation ? [a] : []), a, b, b].map(key => `Bearer ${key}`)); + // B never receives A's continuation or reasoning, and the follow-up on B keeps B's + // continuation and its reasoning replay instead of stripping it as foreign. + expect(builds).toEqual([ + { key: a }, + ...(continuation ? [{ key: a, continuation: "private-a" }] : []), + { key: a, continuation: "private-a" }, + { key: b, stripReasoning: true }, + { key: b, continuation: "private-b" }, + ]); + expect(phases).toEqual([]); + }); }); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d0b3f1a88b4..6e3f519b36b 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -673,7 +673,7 @@ "gemini-inline.test.ts": "images", "gemini-web-search.test.ts": "adapters/google", "generic-session-affinity.test.ts": "oauth", "server-generic-oauth-affinity.test.ts": "server", - "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", + "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", "github-copilot-catalog-hints.test.ts": "providers/github-copilot", diff --git a/tests/oauth/generic-oauth-combo-account-cooldown.test.ts b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts new file mode 100644 index 00000000000..0ec326ed748 --- /dev/null +++ b/tests/oauth/generic-oauth-combo-account-cooldown.test.ts @@ -0,0 +1,131 @@ +/** + * A generic OAuth pool's spent account must not park the whole combo target. + * + * The per-request account budget stops the pool's own rotation before it reaches every stored + * account, so a 429 can reach the combo layer while a sibling is still healthy. Cooling the target + * for that account's `Retry-After` (up to the 24h server-delay ceiling) would answer zero-attempt + * 503s, or pay for a fallback target, while the pool still has an account that answers. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearComboSelectionState, clearComboTargetCooldowns, isComboTargetInCooldown } from "../../src/combos"; +import { clearGenericFailoverHealth, eligibleFailoverAccounts } from "../../src/oauth/generic-account-failover"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; +import { clearAccountQuotaCache } from "../../src/providers/quota"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { handleResponses } from "../../src/server/responses"; +import type { OcxConfig } from "../../src/types"; +import { chatSuccess } from "../helpers/combo-failover-upstream"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const TARGET = { provider: "xai", model: "m1" }; +const originalHome = process.env.OPENCODEX_HOME; +const originalFetch = globalThis.fetch; +let home = ""; +let releaseSpendHome: (() => void) | undefined; +/** Account index per upstream send (`B` for the fallback target). */ +let sends: string[] = []; + +async function seed(count: number): Promise { + for (let index = 1; index <= count; index++) { + await saveCredential("xai", { + access: `synthetic-xai-access-${index}`, refresh: `synthetic-xai-refresh-${index}`, + expires: Date.now() + 3_600_000, accountId: `synthetic-xai-${index}`, + } as never, { addAccount: true }); + } + await setActiveAccount("xai", getAccountSet("xai")!.accounts[0]!.id); +} + +/** Accounts at or below `spentThrough` answer a one-hour 429; the rest answer 200. */ +function upstream(spentThrough: number): void { + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input instanceof Request ? input.url : input); + if (url.startsWith("https://fallback.test/")) { + sends.push("B"); + return chatSuccess("fallback answer", "m2"); + } + const index = Number(new Headers(init?.headers).get("authorization")?.split("-").pop()); + sends.push(String(index)); + return index <= spentThrough + ? Response.json({ error: { message: "synthetic rate limit" } }, { status: 429, headers: { "retry-after": "3600" } }) + : chatSuccess("pool answer", "m1"); + }) as typeof fetch; +} + +const config: OcxConfig = { + port: 0, defaultProvider: "xai", + providers: { + xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "oauth", models: ["m1"] }, + fallback: { adapter: "openai-chat", baseUrl: "https://fallback.test/v1", authMode: "key", apiKey: "synthetic-fallback-key", models: ["m2"] }, + }, + combos: { pooled: { strategy: "failover", targets: [TARGET, { provider: "fallback", model: "m2" }] } }, +}; + +async function post(): Promise { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/pooled", input: "Answer briefly", stream: false }), + }), config, { model: "", provider: "" }); + expect(response.status).toBe(200); + return response.text(); +} + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-generic-combo-cooldown-")); + process.env.OPENCODEX_HOME = home; + sends = []; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + clearResponseStateForTests(); + releaseSpendHome = acquireOwnedSpendHome(); +}); + +afterEach(async () => { + await flushResponseState(); + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + clearResponseStateForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + if (home) removeTreeWithRetry(home); +}); + +test("a spent generic OAuth account cools the account, not the combo target", async () => { + await seed(5); + upstream(4); + + expect(await post()).toContain("fallback answer"); + // Precondition: the request budget ended the pool's rotation before the healthy fifth account. + expect(sends).not.toContain("5"); + expect(sends.at(-1)).toBe("B"); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(false); + // Every account this request reached is cooled by the pool, the last one included. + expect(eligibleFailoverAccounts("xai")).toEqual([getAccountSet("xai")!.accounts[4]!.id]); + + // The pool, not the fallback target, serves the next request. Without proactive preference the + // pool may still open on its cooled active account once before rotating; that is the pool's own + // contract and stays bounded by its rotation. + sends = []; + expect(await post()).toContain("pool answer"); + expect(sends).not.toContain("B"); + expect(sends.at(-1)).toBe("5"); +}, 20_000); + +test("every generic OAuth account spent still cools the combo target", async () => { + await seed(2); + upstream(2); + + expect(await post()).toContain("fallback answer"); + expect(sends).toEqual(["1", "2", "B"]); + expect(isComboTargetInCooldown("pooled", TARGET)).toBe(true); +}, 20_000); diff --git a/tests/oauth/generic-oauth-failover.test.ts b/tests/oauth/generic-oauth-failover.test.ts index bb4150200f5..63a253f5c60 100644 --- a/tests/oauth/generic-oauth-failover.test.ts +++ b/tests/oauth/generic-oauth-failover.test.ts @@ -559,8 +559,11 @@ describe("sidecar on429 wiring", () => { // The counts differ by rotator because the recovery sites differ, and each number is a // statement about which providers can recover where: // - // generic = 5: streaming loop, continuation loop, sidecar hook, runTurn preflight, - // native Responses passthrough. The Antigravity 403 verify-account + // generic = 6: streaming loop, continuation loop, sidecar hook, runTurn preflight, + // native Responses passthrough, plus the combo failure attribution in + // core-combo.ts, which sends nothing: it records the refused account the + // request budget left uncooled and spares the combo target only when the + // rotator names a sibling. The Antigravity 403 verify-account // arm deliberately does NOT use this rotator: a verification refusal // must not record rate-limit cooldown semantics, so it moves via // rotateAntigravityAccountOnAuthRefusal instead. The new default @@ -581,7 +584,7 @@ describe("sidecar on429 wiring", () => { // // Adding a recovery site means deciding, deliberately, which rotators it needs and // updating the matching number. That decision is the thing this test exists to force. - expect(counts.generic).toBe(5); + expect(counts.generic).toBe(6); expect(counts.anthropic).toBe(3); expect(counts.key).toBe(7); }); diff --git a/tests/responses/jev-initial-effort-wire.test.ts b/tests/responses/jev-initial-effort-wire.test.ts index 513aeb406df..1886db30755 100644 --- a/tests/responses/jev-initial-effort-wire.test.ts +++ b/tests/responses/jev-initial-effort-wire.test.ts @@ -230,6 +230,20 @@ test("ordinary force invariant is not relaxed", () => { .toThrow("force combo default effort requires a valid defaultEffort"); }); +test("a JEV combo whose only allowlist is unsupported names that target in its 503", async () => { + const f = fixture("responses", "model", "high"); + f.config.providers.a!.reasoningEfforts = []; + f.config.combos!.auto!.targets[0]!.reasoningEfforts = ["high"]; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/auto", input: "Solve the fixture task.", stream: false }), + }), f.config, { model: "", provider: "" }, {}); + expect(response.status).toBe(503); + expect(((await response.json()) as { error: { message: string } }).error.message) + .toBe("No available targets for combo: auto (a/m: ineligible for this request)"); + expect(f.inference.bodies).toHaveLength(0); +}); + for (const lane of ["native", "responses"] as const) { test(`${lane}: unallowlisted service effort fails open without applying the rejected effort`, async () => { const f = fixture(lane, "typesafe", "high"); diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index fc36df90b15..c127c0ae2c0 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -321,6 +321,18 @@ describe("Responses account usage attribution", () => { expect(bearers).toEqual(["Bearer pool-a-access-token", "Bearer pool-b-access-token"]); expect(logCtx.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); expect(logCtx.activeAttempt?.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); + // The move is a named recovery: the first account's row keeps its real 429 rather than an + // inferred 502, and the second send says why it exists. + expect(logCtx.attempts?.map(attempt => ({ + label: attempt.accountLogLabel, + status: attempt.status, + sendCount: attempt.sendCount, + recoveryKinds: attempt.recoveryKinds, + failureCause: attempt.failureCause, + }))).toEqual([ + { label: fallbackCodexAccountLogLabel("pool-a"), status: 429, sendCount: 1, recoveryKinds: [], failureCause: "rate-limit" }, + { label: fallbackCodexAccountLogLabel("pool-b"), status: 0, sendCount: 1, recoveryKinds: ["oauth-account-429"], failureCause: undefined }, + ]); }); }); diff --git a/tests/responses/responses-combo-exhausted-error.test.ts b/tests/responses/responses-combo-exhausted-error.test.ts index de4494c5e98..c0c1410004c 100644 --- a/tests/responses/responses-combo-exhausted-error.test.ts +++ b/tests/responses/responses-combo-exhausted-error.test.ts @@ -1,6 +1,8 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { clearComboSelectionState, clearComboTargetCooldowns, coolComboTarget, snapshotComboQuotaCooldowns } from "../../src/combos"; import { clearKeyCooldowns } from "../../src/providers/key-failover"; +import { clearCachedProviderQuotas, setCachedProviderQuotaForTests } from "../../src/providers/quota-routing-cache"; +import type { ProviderQuota } from "../../src/providers/quota-types"; import { handleComboResponses, handleResponses } from "../../src/server/responses/core"; import { executeComboResponses } from "../../src/server/responses/core-combo"; import { createTranslatorBudget } from "../../src/lib/translator-budget"; @@ -288,3 +290,43 @@ describe("combo exhaustion reports the primary's quota refusal", () => { expect(response.status).toBe(502); }); }); + +describe("zero-attempt combo refusal names why each target was excluded", () => { + afterEach(() => clearCachedProviderQuotas()); + + test("missing, disabled, quota-spent, cooling and request-ineligible targets each carry a reason", async () => { + setCachedProviderQuotaForTests("spent", { updatedAt: Date.now(), weeklyPercent: 100 } as ProviderQuota); + coolComboTarget("all", { provider: "cooling", model: "model-cooling" }, { retryAfter: "42", status: 429 }); + const cfg = { + defaultProvider: "spent", + providers: { + off: { ...provider("off"), disabled: true }, + spent: provider("spent"), + cooling: provider("cooling"), + metered: provider("metered"), + }, + combos: { + all: { + strategy: "failover", + targets: [ + { provider: "gone", model: "model-gone" }, + { provider: "off", model: "model-off" }, + { provider: "spent", model: "model-spent" }, + { provider: "cooling", model: "model-cooling" }, + { provider: "metered", model: "model-metered", metered: true }, + ], + }, + }, + } as unknown as OcxConfig; + globalThis.fetch = (async () => { throw new Error("no target may be dispatched"); }) as unknown as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/all", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" } as RequestLogContext); + expect(response.status).toBe(503); + const { error } = await response.json() as { error: { code: string; message: string } }; + expect(error.code).toBe("combo_unavailable"); + expect(error.message).toMatch(/^No available targets for combo: all \(gone\/model-gone: provider missing; off\/model-off: provider disabled; spent\/model-spent: quota exhausted; cooling\/model-cooling: cooldown 4[12]s; metered\/model-metered: ineligible for this request\)$/); + }); +}); diff --git a/tests/responses/responses-context-overflow.test.ts b/tests/responses/responses-context-overflow.test.ts index ec81eb872ef..e7bf31c1278 100644 --- a/tests/responses/responses-context-overflow.test.ts +++ b/tests/responses/responses-context-overflow.test.ts @@ -4,11 +4,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../src/config"; import { startServer } from "../../src/server"; +import { handleResponses } from "../../src/server/responses/core"; +import type { RequestLogContext } from "../../src/server/request-log"; import { PROVIDER_INPUT_TOO_LARGE_MESSAGE } from "../../src/server/responses/context-overflow"; import { OLLAMA_LATE_ATTRIBUTION_LIMIT_MESSAGE } from "../../src/adapters/ollama-native"; import { isLateAttributionRefusal } from "../../src/server/responses/core-combo-failure"; import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { clearComboTargetCooldowns, coolComboTarget } from "../../src/combos"; let testDir = ""; let previousOcxHome: string | undefined; @@ -418,3 +422,78 @@ describe("combo handling of the Ollama late-attribution refusal", () => { expect(isLateAttributionRefusal(failure({ upstreamMessage: undefined }))).toBe(false); }); }); + +// Every target refused by the local admission gate means no provider saw the turn. The client +// must hear that from the proxy, with the estimate and each window, instead of the provider +// wording an operator reads as an upstream rejection. +describe("combo whose targets all fail local input admission", () => { + async function runCombo(stream: boolean, second: "local" | "upstream" | "cooled") { + let hits = 0; + const local = upstream413(() => { hits += 1; }); + const first = { ...provider("openai-chat", local), modelContextWindows: { "kimi-k3": 1_000 } }; + const next = config({ + first, + second: second === "upstream" + ? provider("openai-chat", freePromptCap413(() => { hits += 1; })) + : { ...provider("openai-chat", local), modelContextWindows: { "kimi-k3": 2_000 } }, + }); + next.combos = { fallback: { strategy: "failover", targets: [ + { provider: "first", model: "kimi-k3" }, { provider: "second", model: "kimi-k3" }, + ] } }; + clearComboTargetCooldowns(); + if (second === "cooled") coolComboTarget("fallback", { provider: "second", model: "kimi-k3" }, { retryAfter: "120" }); + const logCtx = { model: "", provider: "" } as RequestLogContext; + const release = acquireOwnedSpendHome(); + try { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/fallback", stream, max_output_tokens: 500, input: "x".repeat(20_000) }), + }), next, logCtx); + const error = stream + ? (await responseFailed(response)).error as Record + : (expect(response.status).toBe(413), (await response.json() as { error: Record }).error); + return { error, hits, attempts: logCtx.attempts ?? [], logCtx }; + } finally { + release(); + clearComboTargetCooldowns(); + } + } + + test.each([true, false])("says the proxy refused it, with estimate and windows (stream=%s)", async stream => { + const { error, hits, attempts, logCtx } = await runCombo(stream, "local"); + expect(hits).toBe(0); + expect(error.code).toBe("context_length_exceeded"); + expect(error.type).toBe("invalid_request_error"); + const message = String(error.message); + expect(message).not.toBe(PROVIDER_INPUT_TOO_LARGE_MESSAGE); + expect(message).toContain("refused this turn locally"); + expect(message).toContain("No target of combo fallback that was tried"); + expect(message).toMatch(/~\d+ tokens/); + expect(message).toContain("(1000 tokens)"); + expect(message).toContain("(2000 tokens)"); + expect(attempts.map(attempt => [attempt.status, attempt.sendCount, attempt.errorCode])).toEqual([ + [413, 0, "input_admission_refused"], + [413, 0, "input_admission_refused"], + ]); + expect(logCtx.errorCode).toBe("input_admission_refused"); + }); + + test("a target skipped for cooldown is not claimed to be too small", async () => { + const { error, hits, attempts } = await runCombo(false, "cooled"); + expect(hits).toBe(0); + const message = String(error.message); + expect(message).toContain("No target of combo fallback that was tried can fit its input"); + expect(message).toContain("(1000 tokens)"); + expect(message).not.toContain("(2000 tokens)"); + expect(attempts.map(attempt => attempt.errorCode)).toEqual(["input_admission_refused"]); + }); + + test("a target that reached the provider keeps the provider wording", async () => { + const { error, hits, attempts, logCtx } = await runCombo(false, "upstream"); + expect(hits).toBe(1); + expect(logCtx.errorCode).toBeUndefined(); + expect(error.message).toBe(PROVIDER_INPUT_TOO_LARGE_MESSAGE); + expect(attempts.map(attempt => attempt.errorCode)).toEqual(["input_admission_refused", "http_413"]); + }); +}); diff --git a/tests/server/health-scoring.test.ts b/tests/server/health-scoring.test.ts index b54436cc15c..d1458c1454a 100644 --- a/tests/server/health-scoring.test.ts +++ b/tests/server/health-scoring.test.ts @@ -247,6 +247,60 @@ describe("health-aware scoring (RI-06)", () => { expect(finalTarget.successRate).toBe(1); }); + test("a stream that dies after its 200 head is a failure for the attempt that dropped it", () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + // b/m2 failed over to a/m1, whose 200 stream was then cut mid-flight. + appendUsageEntry({ + ...row("combo-abort", 200, 1500, { + provider: "combo", + model: "c1", + terminalStatus: "incomplete", + timestamp: Date.now() - 1_000, + }), + attempts: [ + { ordinal: 1, provider: "b", model: "m2", adapter: "openai-chat", status: 503, durationMs: 400, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + { ordinal: 2, provider: "a", model: "m1", adapter: "openai-chat", status: 200, durationMs: 1100, streamAborted: true, sendCount: 1, recoveryKinds: [], usageStatus: "unreported" }, + ], + }); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(6); + expect(evidence.failures).toBe(1); + expect(evidence.incompleteStreamRate).toBeCloseTo(1 / 6, 5); + }); + + test("a request with no physical send to a target is not health evidence for it", () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + // Answered locally: the row names a/m1 but nothing left the proxy. + appendUsageEntry({ + ...row("local", 200, 5, { localTerminalReason: "local-answer" }), + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 200, durationMs: 5, locallyAnswered: true, sendCount: 0, recoveryKinds: [], usageStatus: "reported" }, + ], + }); + // Refused before send for a/m1, then served by b/m2. + appendUsageEntry({ + ...row("refused", 200, 900, { provider: "b", model: "m2", timestamp: Date.now() - 1_000 }), + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 1, sendCount: 0, recoveryKinds: [], usageStatus: "unreported" }, + { ordinal: 2, provider: "b", model: "m2", adapter: "openai-chat", status: 200, durationMs: 900, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + }); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(5); + expect(evidence.failures).toBeUndefined(); + expect(healthEvidenceForCandidate({ provider: "b", model: "m2" }).sampleCount).toBe(1); + }); + + test("a local refusal logged before any attempt began is not health evidence", () => { + for (let index = 0; index < 5; index++) appendUsageEntry(row(`ok-${index}`, 200, 800)); + // Refused by the proxy itself (e.g. server_busy admission): no attempts array, and the + // persisted row carries only the derived `local-refusal` cause, not `localTerminalReason`. + appendUsageEntry(row("busy", 503, 2, { failureCause: "local-refusal", timestamp: Date.now() - 1_000 })); + const evidence = healthEvidenceForCandidate({ provider: "a", model: "m1" }); + expect(evidence.sampleCount).toBe(5); + expect(evidence.failures).toBeUndefined(); + }); + test("execution path applies live codex account cooldown to openai candidates", async () => { const { clearCodexUpstreamHealth, recordCodexUpstreamOutcome } = await import("../../src/codex/routing"); clearCodexUpstreamHealth(); diff --git a/tests/server/server-combo-reasoning-replay-eligibility.test.ts b/tests/server/server-combo-reasoning-replay-eligibility.test.ts index 38451a7f950..c9a9a2d2539 100644 --- a/tests/server/server-combo-reasoning-replay-eligibility.test.ts +++ b/tests/server/server-combo-reasoning-replay-eligibility.test.ts @@ -175,7 +175,7 @@ describe("combo mandatory reasoning replay failover", () => { expect(response.status).toBe(503); expect(response.headers.get("content-type")).toBe("application/json"); expect(await response.text()).toBe( - '{"error":{"message":"No available targets for combo: free","type":"server_error","code":"combo_unavailable"}}', + '{"error":{"message":"No available targets for combo: free (missing/bad-model: provider missing)","type":"server_error","code":"combo_unavailable"}}', ); });