diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 53a46fee97d..7e2796d0d78 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1105,6 +1105,7 @@ export const de: Record = { "logs.detail.attempt.recovery.key401": "API-Schlüssel erneut authentifiziert", "logs.detail.attempt.recovery.oauthAccount403": "Kontovalidierung erforderlich (403)", "logs.detail.attempt.recovery.oauthAccount429": "Konto rate-limitiert (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Kontingent des Kontos erschöpft (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Veralteter verschlüsselter Zustand verworfen", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning-Aufwand reduziert", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Schnellmodus abgelehnt, erneut mit Standardtempo versucht", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index dd45d9ef3f7..4f14c346140 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -1163,6 +1163,7 @@ export const en = { "logs.detail.attempt.recovery.key401": "API key re-authentication", "logs.detail.attempt.recovery.oauthAccount403": "Account validation required (403)", "logs.detail.attempt.recovery.oauthAccount429": "Account rate-limited (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Account quota exhausted (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Stale encrypted state dropped", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning effort downgraded", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Fast mode refused, retried at standard speed", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index f913bf03e23..498a2164657 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -1139,6 +1139,7 @@ export const fr: Record = { "logs.detail.attempt.recovery.key401": "Ré-authentification de la clé API", "logs.detail.attempt.recovery.oauthAccount403": "Validation du compte requise (403)", "logs.detail.attempt.recovery.oauthAccount429": "Compte limité (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Quota du compte épuisé (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "État chiffré obsolète abandonné", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Effort de raisonnement réduit", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Mode rapide refusé, nouvel essai à vitesse standard", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 3dde661880a..f615953cc56 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1046,6 +1046,7 @@ export const ja: Record = { "logs.detail.attempt.recovery.key401": "API キーの再認証", "logs.detail.attempt.recovery.oauthAccount403": "アカウントの検証が必要 (403)", "logs.detail.attempt.recovery.oauthAccount429": "アカウントのレート制限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "アカウントのクォータ枯渇 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "古い暗号化状態を破棄", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "推論強度を下げて再試行", "logs.detail.attempt.recovery.anthropicFastDowngrade": "高速モードが拒否され、標準速度で再試行", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 44652dd9c54..ecfb9e48352 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1144,6 +1144,7 @@ export const ko: Record = { "logs.detail.attempt.recovery.key401": "API 키 재인증", "logs.detail.attempt.recovery.oauthAccount403": "계정 확인 필요 (403)", "logs.detail.attempt.recovery.oauthAccount429": "계정 속도 제한 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "계정 할당량 소진 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "만료된 암호화 상태 제거", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "추론 강도 하향", "logs.detail.attempt.recovery.anthropicFastDowngrade": "빠른 모드 거부, 표준 속도로 재시도", diff --git a/gui/src/i18n/pt.ts b/gui/src/i18n/pt.ts index 39477bb4852..9e54d3e4190 100644 --- a/gui/src/i18n/pt.ts +++ b/gui/src/i18n/pt.ts @@ -1158,6 +1158,7 @@ export const pt: Record = { "logs.detail.attempt.recovery.key401": "Reautenticação da chave de API", "logs.detail.attempt.recovery.oauthAccount403": "Validação da conta necessária (403)", "logs.detail.attempt.recovery.oauthAccount429": "Conta com limite de taxa (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Cota da conta esgotada (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Estado criptografado obsoleto descartado", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Esforço de raciocínio reduzido", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Modo rápido recusado, repetido em velocidade padrão", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index bf5ac713e68..08d8f95cad9 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1122,6 +1122,7 @@ export const ru: Record = { "logs.detail.attempt.recovery.key401": "Повторная аутентификация API-ключа", "logs.detail.attempt.recovery.oauthAccount403": "Требуется проверка аккаунта (403)", "logs.detail.attempt.recovery.oauthAccount429": "Ограничение частоты для аккаунта (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Квота аккаунта исчерпана (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Устаревшее зашифрованное состояние отброшено", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Уровень рассуждения понижен", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Быстрый режим отклонён, повтор со стандартной скоростью", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 72e3c72180c..104d49e9769 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -1149,6 +1149,7 @@ export const tr: Record = { "logs.detail.attempt.recovery.key401": "API anahtarı yeniden doğrulandı", "logs.detail.attempt.recovery.oauthAccount403": "Hesap doğrulaması gerekli (403)", "logs.detail.attempt.recovery.oauthAccount429": "Hesap hız sınırına takıldı (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Hesap kotası tükendi (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Eski şifreli durum bırakıldı", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Akıl yürütme düzeyi düşürüldü", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Hızlı mod reddedildi, standart hızda yeniden denendi", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index b492db00252..5a5860cdb61 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -1143,6 +1143,7 @@ export const vi: Record = { "logs.detail.attempt.recovery.key401": "Xác thực lại khóa API", "logs.detail.attempt.recovery.oauthAccount403": "Cần xác minh tài khoản (403)", "logs.detail.attempt.recovery.oauthAccount429": "Tài khoản bị giới hạn tần suất (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Tài khoản đã hết hạn mức (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Đã bỏ trạng thái mã hóa cũ", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Đã giảm mức suy luận", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Chế độ nhanh bị từ chối, thử lại ở tốc độ chuẩn", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index aa8e6d5f8ac..cde4139086c 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -2948,6 +2948,7 @@ export const zhTW: Record = { "logs.detail.attempt.recovery.key401": "API 金鑰重新驗證", "logs.detail.attempt.recovery.oauthAccount403": "需要驗證帳號 (403)", "logs.detail.attempt.recovery.oauthAccount429": "帳號速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "帳號配額已用盡 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已捨棄過期加密狀態", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理強度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式遭拒,改以標準速度重試", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 1d24a0df5a6..8302df5a4c9 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1117,6 +1117,7 @@ export const zh: Record = { "logs.detail.attempt.recovery.key401": "API 密钥重新认证", "logs.detail.attempt.recovery.oauthAccount403": "需要验证账号 (403)", "logs.detail.attempt.recovery.oauthAccount429": "账号速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "账号配额已用尽 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已丢弃过期加密状态", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理强度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式被拒,按标准速度重试", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 922634b69ec..a1f5858f394 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -355,6 +355,7 @@ const RECOVERY_KIND_KEYS = { "rate-limit-429": "logs.detail.attempt.recovery.rateLimit429", "anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429", "oauth-account-429": "logs.detail.attempt.recovery.oauthAccount429", + "oauth-account-402": "logs.detail.attempt.recovery.oauthAccount402", "image-413": "logs.detail.attempt.recovery.image413", "empty-completion": "logs.detail.attempt.recovery.emptyCompletion", "console-go-upload-retry": "logs.detail.attempt.recovery.consoleGoUpload", diff --git a/src/lib/request-failure-model.ts b/src/lib/request-failure-model.ts index 7ac0cb9ba61..dc924ac2874 100644 --- a/src/lib/request-failure-model.ts +++ b/src/lib/request-failure-model.ts @@ -223,6 +223,8 @@ const RECOVERY_KIND_CAUSE = { "rate-limit-429": "rate-limit", "anthropic-oauth-429": "rate-limit", "oauth-account-429": "rate-limit", + // A 402 is the account's quota, not a transient limit; only another account can serve it. + "oauth-account-402": "quota-exhausted", "image-413": "payload-too-large", // The gateway rejects a body it accepts seconds later and the replay is byte-identical, so // nothing about the payload was wrong; the origin declined to take it at that moment. diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts index 5c4411c3b8c..3eb2e540a14 100644 --- a/src/routing/analytics.ts +++ b/src/routing/analytics.ts @@ -122,6 +122,7 @@ const COOLDOWN_RECOVERY_KINDS = new Set([ "oauth-account-403", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", ]); function percentile(sorted: number[], p: number): number | undefined { diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 17071ba24d2..6ae1b20d15b 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -32,6 +32,7 @@ import { type CodexWsPreOutputRefusalHold, } from "./codex-ws-wire"; import type { RequestLogContext } from "../request-log"; +import type { AttemptRecoveryKind } from "../../usage/log"; import type { DataPlaneAdmission } from "../auth-cors"; import type { InboundWire } from "../../providers/registry"; import type { BunRuntimeGateInput } from "./ws-upstream"; @@ -971,6 +972,10 @@ export async function retryCodexPoolOnAlternateAccount( // What this diff does fix is that the rungs are now CHARGED instead of free. const maxRetrySends = retrySameConfirmedAccount ? 7 : 1; let retrySendCount = 0; + // The gated-model 400 has no roster kind; it stays unattributed as before. + const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 ? "oauth-account-429" + : outcomeStatus === 402 ? "oauth-account-402" + : isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined; let upstreamResponse: Response; try { while (true) { @@ -1005,7 +1010,10 @@ export async function retryCodexPoolOnAlternateAccount( // The move is a physical send like any other, so the root workflow is charged too. chargeWorkflowSends(args.options.workflowRootId, 1); } - noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate); + // A move seals the first account's attempt here. Give it the status that account actually + // answered, not the 502 the split infers, and name why the second send exists. + if (logCtx.activeAttempt && logCtx.activeAttempt.status < 100) logCtx.activeAttempt.status = firstResponse.status; + noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate, moveRecovery); try { upstreamResponse = await fetchWithHeaderTimeout( request.url, diff --git a/src/usage/telemetry-contract.ts b/src/usage/telemetry-contract.ts index 157d372f7c7..69247569cdf 100644 --- a/src/usage/telemetry-contract.ts +++ b/src/usage/telemetry-contract.ts @@ -32,6 +32,7 @@ export const ATTEMPT_RECOVERY_KIND_ROSTER = Object.freeze([ "rate-limit-429", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", "image-413", "console-go-upload-retry", "opaque-blob-rejection", diff --git a/tests/lib/failure-stage-model.test.ts b/tests/lib/failure-stage-model.test.ts index 893f85b6178..078942991fd 100644 --- a/tests/lib/failure-stage-model.test.ts +++ b/tests/lib/failure-stage-model.test.ts @@ -199,8 +199,10 @@ describe("recovery kinds speak the shared dictionary", () => { * the durable log were forbidden and unfunded. */ test("no recorded recovery classifies as a resend that cannot help", () => { + // Except the Codex 402 account move: re-asking the exhausted account is futile, and the move + // to another account is a route decision this table explicitly does not make. const futile = ATTEMPT_RECOVERY_KIND_ROSTER .filter(kind => causeDisposition(causeForRecoveryKind(kind)) === "resend-is-futile"); - expect(futile).toEqual([]); + expect(futile).toEqual(["oauth-account-402"]); }); }); diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index fc36df90b15..a88745c707c 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -11,7 +11,8 @@ import { clearThreadAccountMap, getCodexUpstreamHealth, } from "../../src/codex/routing"; -import type { RequestLogContext } from "../../src/server/request-log"; +import { addFinalRequestLog, type RequestLogContext } from "../../src/server/request-log"; +import { createRequestMetricsOwner } from "../../src/server/request-metrics"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -321,6 +322,49 @@ describe("Responses account usage attribution", () => { expect(bearers).toEqual(["Bearer pool-a-access-token", "Bearer pool-b-access-token"]); expect(logCtx.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); expect(logCtx.activeAttempt?.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); + // The move is a named recovery: the first account's row keeps its real 429 rather than an + // inferred 502, and the second send says why it exists. + expect(logCtx.attempts?.map(attempt => ({ + label: attempt.accountLogLabel, + status: attempt.status, + sendCount: attempt.sendCount, + recoveryKinds: attempt.recoveryKinds, + failureCause: attempt.failureCause, + }))).toEqual([ + { label: fallbackCodexAccountLogLabel("pool-a"), status: 429, sendCount: 1, recoveryKinds: [], failureCause: "rate-limit" }, + { label: fallbackCodexAccountLogLabel("pool-b"), status: 0, sendCount: 1, recoveryKinds: ["oauth-account-429"], failureCause: undefined }, + ]); + }); + }); + + test("a pre-stream 402 move is recorded and counted as a quota recovery, not a rate limit", async () => { + await withPoolHome(async () => { + takeSpendHome(); + const config = poolConfig(["pool-a", "pool-b"]); + for (const id of ["pool-a", "pool-b"]) { + savePoolCredential(id); + updateAccountQuota(id, id === "pool-a" ? 10 : 20); + } + let sends = 0; + globalThis.fetch = (async () => ++sends === 1 + ? Response.json({ error: { message: "payment required" } }, { status: 402 }) + : completedResponse("pool-b-response")) as typeof fetch; + + const metrics = createRequestMetricsOwner(123); + const logCtx: RequestLogContext = { model: "", provider: "", inboundProtocol: "responses", requestMetricsRecorder: metrics }; + const response = await handleResponses(request(), config, logCtx, {}); + expect(response.status).toBe(200); + expect(logCtx.attempts?.map(attempt => [attempt.status, attempt.recoveryKinds])).toEqual([ + [402, []], + [0, ["oauth-account-402"]], + ]); + + addFinalRequestLog("codex-402-move", Date.now() - 1_000, logCtx, 200, { + terminalStatus: "completed", closeReason: "terminal", + }, () => {}); + const recoveries = metrics.snapshot().split("\n").filter(line => line.startsWith("opencodex_recoveries_total{")); + expect(recoveries.find(line => line.includes('recovery="quota"'))?.endsWith(" 1")).toBe(true); + expect(recoveries.find(line => line.includes('recovery="rate_limit"'))?.endsWith(" 0") ?? true).toBe(true); }); });