From cd5409daf81e33c010830e5bfbd6542c5bd6412d Mon Sep 17 00:00:00 2001 From: Vadym O Date: Thu, 15 Oct 2026 21:38:08 +0800 Subject: [PATCH 1/2] fix(codex): attribute a pool account move to its recovery and real status The alternate-account send passed no recovery kind, so the second send on the row had no cause and the first account's sealed attempt showed an inferred 502 instead of the status it answered (e.g. 429). Reuse the roster kinds: oauth-account-429 for quota (429/402), transient-5xx for a 5xx move. Co-Authored-By: Claude Code --- src/server/responses/core-codex-account.ts | 10 +++++++++- tests/responses/responses-account-label.test.ts | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 17071ba24d2..bad576636da 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -32,6 +32,7 @@ import { type CodexWsPreOutputRefusalHold, } from "./codex-ws-wire"; import type { RequestLogContext } from "../request-log"; +import type { AttemptRecoveryKind } from "../../usage/log"; import type { DataPlaneAdmission } from "../auth-cors"; import type { InboundWire } from "../../providers/registry"; import type { BunRuntimeGateInput } from "./ws-upstream"; @@ -971,6 +972,10 @@ export async function retryCodexPoolOnAlternateAccount( // What this diff does fix is that the rungs are now CHARGED instead of free. const maxRetrySends = retrySameConfirmedAccount ? 7 : 1; let retrySendCount = 0; + // The gated-model 400 has no roster kind; it stays unattributed as before. + const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 || outcomeStatus === 402 + ? "oauth-account-429" + : isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined; let upstreamResponse: Response; try { while (true) { @@ -1005,7 +1010,10 @@ export async function retryCodexPoolOnAlternateAccount( // The move is a physical send like any other, so the root workflow is charged too. chargeWorkflowSends(args.options.workflowRootId, 1); } - noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate); + // A move seals the first account's attempt here. Give it the status that account actually + // answered, not the 502 the split infers, and name why the second send exists. + if (logCtx.activeAttempt && logCtx.activeAttempt.status < 100) logCtx.activeAttempt.status = firstResponse.status; + noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate, moveRecovery); try { upstreamResponse = await fetchWithHeaderTimeout( request.url, diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index fc36df90b15..c127c0ae2c0 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -321,6 +321,18 @@ describe("Responses account usage attribution", () => { expect(bearers).toEqual(["Bearer pool-a-access-token", "Bearer pool-b-access-token"]); expect(logCtx.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); expect(logCtx.activeAttempt?.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b")); + // The move is a named recovery: the first account's row keeps its real 429 rather than an + // inferred 502, and the second send says why it exists. + expect(logCtx.attempts?.map(attempt => ({ + label: attempt.accountLogLabel, + status: attempt.status, + sendCount: attempt.sendCount, + recoveryKinds: attempt.recoveryKinds, + failureCause: attempt.failureCause, + }))).toEqual([ + { label: fallbackCodexAccountLogLabel("pool-a"), status: 429, sendCount: 1, recoveryKinds: [], failureCause: "rate-limit" }, + { label: fallbackCodexAccountLogLabel("pool-b"), status: 0, sendCount: 1, recoveryKinds: ["oauth-account-429"], failureCause: undefined }, + ]); }); }); From c24597d5135f8ce976f1b1f6d1160e7870115695 Mon Sep 17 00:00:00 2001 From: Vadym O Date: Fri, 16 Oct 2026 11:37:00 +0800 Subject: [PATCH 2/2] fix(codex): record a 402 account move as a quota recovery A Codex 402 move was recorded as oauth-account-429, so request metrics counted it as rate_limit and the dashboard labelled it an OAuth 429. Add oauth-account-402 mapped to quota-exhausted and wire it into the cooldown set, the dashboard label map and all locale catalogs. Co-Authored-By: Claude Code --- gui/src/i18n/de.ts | 1 + gui/src/i18n/en.ts | 1 + gui/src/i18n/fr.ts | 1 + gui/src/i18n/ja.ts | 1 + gui/src/i18n/ko.ts | 1 + gui/src/i18n/pt.ts | 1 + gui/src/i18n/ru.ts | 1 + gui/src/i18n/tr.ts | 1 + gui/src/i18n/vi.ts | 1 + gui/src/i18n/zh-TW.ts | 1 + gui/src/i18n/zh.ts | 1 + gui/src/pages/Logs.tsx | 1 + src/lib/request-failure-model.ts | 2 ++ src/routing/analytics.ts | 1 + src/server/responses/core-codex-account.ts | 4 +-- src/usage/telemetry-contract.ts | 1 + tests/lib/failure-stage-model.test.ts | 4 ++- .../responses/responses-account-label.test.ts | 34 ++++++++++++++++++- 18 files changed, 54 insertions(+), 4 deletions(-) diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 53a46fee97d..7e2796d0d78 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1105,6 +1105,7 @@ export const de: Record = { "logs.detail.attempt.recovery.key401": "API-Schlüssel erneut authentifiziert", "logs.detail.attempt.recovery.oauthAccount403": "Kontovalidierung erforderlich (403)", "logs.detail.attempt.recovery.oauthAccount429": "Konto rate-limitiert (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Kontingent des Kontos erschöpft (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Veralteter verschlüsselter Zustand verworfen", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning-Aufwand reduziert", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Schnellmodus abgelehnt, erneut mit Standardtempo versucht", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index dd45d9ef3f7..4f14c346140 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -1163,6 +1163,7 @@ export const en = { "logs.detail.attempt.recovery.key401": "API key re-authentication", "logs.detail.attempt.recovery.oauthAccount403": "Account validation required (403)", "logs.detail.attempt.recovery.oauthAccount429": "Account rate-limited (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Account quota exhausted (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Stale encrypted state dropped", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning effort downgraded", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Fast mode refused, retried at standard speed", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index f913bf03e23..498a2164657 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -1139,6 +1139,7 @@ export const fr: Record = { "logs.detail.attempt.recovery.key401": "Ré-authentification de la clé API", "logs.detail.attempt.recovery.oauthAccount403": "Validation du compte requise (403)", "logs.detail.attempt.recovery.oauthAccount429": "Compte limité (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Quota du compte épuisé (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "État chiffré obsolète abandonné", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Effort de raisonnement réduit", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Mode rapide refusé, nouvel essai à vitesse standard", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 3dde661880a..f615953cc56 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1046,6 +1046,7 @@ export const ja: Record = { "logs.detail.attempt.recovery.key401": "API キーの再認証", "logs.detail.attempt.recovery.oauthAccount403": "アカウントの検証が必要 (403)", "logs.detail.attempt.recovery.oauthAccount429": "アカウントのレート制限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "アカウントのクォータ枯渇 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "古い暗号化状態を破棄", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "推論強度を下げて再試行", "logs.detail.attempt.recovery.anthropicFastDowngrade": "高速モードが拒否され、標準速度で再試行", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 44652dd9c54..ecfb9e48352 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1144,6 +1144,7 @@ export const ko: Record = { "logs.detail.attempt.recovery.key401": "API 키 재인증", "logs.detail.attempt.recovery.oauthAccount403": "계정 확인 필요 (403)", "logs.detail.attempt.recovery.oauthAccount429": "계정 속도 제한 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "계정 할당량 소진 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "만료된 암호화 상태 제거", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "추론 강도 하향", "logs.detail.attempt.recovery.anthropicFastDowngrade": "빠른 모드 거부, 표준 속도로 재시도", diff --git a/gui/src/i18n/pt.ts b/gui/src/i18n/pt.ts index 39477bb4852..9e54d3e4190 100644 --- a/gui/src/i18n/pt.ts +++ b/gui/src/i18n/pt.ts @@ -1158,6 +1158,7 @@ export const pt: Record = { "logs.detail.attempt.recovery.key401": "Reautenticação da chave de API", "logs.detail.attempt.recovery.oauthAccount403": "Validação da conta necessária (403)", "logs.detail.attempt.recovery.oauthAccount429": "Conta com limite de taxa (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Cota da conta esgotada (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Estado criptografado obsoleto descartado", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Esforço de raciocínio reduzido", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Modo rápido recusado, repetido em velocidade padrão", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index bf5ac713e68..08d8f95cad9 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1122,6 +1122,7 @@ export const ru: Record = { "logs.detail.attempt.recovery.key401": "Повторная аутентификация API-ключа", "logs.detail.attempt.recovery.oauthAccount403": "Требуется проверка аккаунта (403)", "logs.detail.attempt.recovery.oauthAccount429": "Ограничение частоты для аккаунта (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Квота аккаунта исчерпана (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Устаревшее зашифрованное состояние отброшено", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Уровень рассуждения понижен", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Быстрый режим отклонён, повтор со стандартной скоростью", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 72e3c72180c..104d49e9769 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -1149,6 +1149,7 @@ export const tr: Record = { "logs.detail.attempt.recovery.key401": "API anahtarı yeniden doğrulandı", "logs.detail.attempt.recovery.oauthAccount403": "Hesap doğrulaması gerekli (403)", "logs.detail.attempt.recovery.oauthAccount429": "Hesap hız sınırına takıldı (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Hesap kotası tükendi (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Eski şifreli durum bırakıldı", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Akıl yürütme düzeyi düşürüldü", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Hızlı mod reddedildi, standart hızda yeniden denendi", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index b492db00252..5a5860cdb61 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -1143,6 +1143,7 @@ export const vi: Record = { "logs.detail.attempt.recovery.key401": "Xác thực lại khóa API", "logs.detail.attempt.recovery.oauthAccount403": "Cần xác minh tài khoản (403)", "logs.detail.attempt.recovery.oauthAccount429": "Tài khoản bị giới hạn tần suất (429)", + "logs.detail.attempt.recovery.oauthAccount402": "Tài khoản đã hết hạn mức (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "Đã bỏ trạng thái mã hóa cũ", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "Đã giảm mức suy luận", "logs.detail.attempt.recovery.anthropicFastDowngrade": "Chế độ nhanh bị từ chối, thử lại ở tốc độ chuẩn", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index aa8e6d5f8ac..cde4139086c 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -2948,6 +2948,7 @@ export const zhTW: Record = { "logs.detail.attempt.recovery.key401": "API 金鑰重新驗證", "logs.detail.attempt.recovery.oauthAccount403": "需要驗證帳號 (403)", "logs.detail.attempt.recovery.oauthAccount429": "帳號速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "帳號配額已用盡 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已捨棄過期加密狀態", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理強度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式遭拒，改以標準速度重試", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 1d24a0df5a6..8302df5a4c9 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1117,6 +1117,7 @@ export const zh: Record = { "logs.detail.attempt.recovery.key401": "API 密钥重新认证", "logs.detail.attempt.recovery.oauthAccount403": "需要验证账号 (403)", "logs.detail.attempt.recovery.oauthAccount429": "账号速率受限 (429)", + "logs.detail.attempt.recovery.oauthAccount402": "账号配额已用尽 (402)", "logs.detail.attempt.recovery.opaqueBlobRejection": "已丢弃过期加密状态", "logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理强度", "logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式被拒，按标准速度重试", diff --git a/gui/src/pages/Logs.tsx b/gui/src/pages/Logs.tsx index 922634b69ec..a1f5858f394 100644 --- a/gui/src/pages/Logs.tsx +++ b/gui/src/pages/Logs.tsx @@ -355,6 +355,7 @@ const RECOVERY_KIND_KEYS = { "rate-limit-429": "logs.detail.attempt.recovery.rateLimit429", "anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429", "oauth-account-429": "logs.detail.attempt.recovery.oauthAccount429", + "oauth-account-402": "logs.detail.attempt.recovery.oauthAccount402", "image-413": "logs.detail.attempt.recovery.image413", "empty-completion": "logs.detail.attempt.recovery.emptyCompletion", "console-go-upload-retry": "logs.detail.attempt.recovery.consoleGoUpload", diff --git a/src/lib/request-failure-model.ts b/src/lib/request-failure-model.ts index 7ac0cb9ba61..dc924ac2874 100644 --- a/src/lib/request-failure-model.ts +++ b/src/lib/request-failure-model.ts @@ -223,6 +223,8 @@ const RECOVERY_KIND_CAUSE = { "rate-limit-429": "rate-limit", "anthropic-oauth-429": "rate-limit", "oauth-account-429": "rate-limit", + // A 402 is the account's quota, not a transient limit; only another account can serve it. + "oauth-account-402": "quota-exhausted", "image-413": "payload-too-large", // The gateway rejects a body it accepts seconds later and the replay is byte-identical, so // nothing about the payload was wrong; the origin declined to take it at that moment. diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts index 5c4411c3b8c..3eb2e540a14 100644 --- a/src/routing/analytics.ts +++ b/src/routing/analytics.ts @@ -122,6 +122,7 @@ const COOLDOWN_RECOVERY_KINDS = new Set([ "oauth-account-403", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", ]); function percentile(sorted: number[], p: number): number | undefined { diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index bad576636da..6ae1b20d15b 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -973,8 +973,8 @@ export async function retryCodexPoolOnAlternateAccount( const maxRetrySends = retrySameConfirmedAccount ? 7 : 1; let retrySendCount = 0; // The gated-model 400 has no roster kind; it stays unattributed as before. - const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 || outcomeStatus === 402 - ? "oauth-account-429" + const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 ? "oauth-account-429" + : outcomeStatus === 402 ? "oauth-account-402" : isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined; let upstreamResponse: Response; try { diff --git a/src/usage/telemetry-contract.ts b/src/usage/telemetry-contract.ts index 157d372f7c7..69247569cdf 100644 --- a/src/usage/telemetry-contract.ts +++ b/src/usage/telemetry-contract.ts @@ -32,6 +32,7 @@ export const ATTEMPT_RECOVERY_KIND_ROSTER = Object.freeze([ "rate-limit-429", "anthropic-oauth-429", "oauth-account-429", + "oauth-account-402", "image-413", "console-go-upload-retry", "opaque-blob-rejection", diff --git a/tests/lib/failure-stage-model.test.ts b/tests/lib/failure-stage-model.test.ts index 893f85b6178..078942991fd 100644 --- a/tests/lib/failure-stage-model.test.ts +++ b/tests/lib/failure-stage-model.test.ts @@ -199,8 +199,10 @@ describe("recovery kinds speak the shared dictionary", () => { * the durable log were forbidden and unfunded. */ test("no recorded recovery classifies as a resend that cannot help", () => { + // Except the Codex 402 account move: re-asking the exhausted account is futile, and the move + // to another account is a route decision this table explicitly does not make. const futile = ATTEMPT_RECOVERY_KIND_ROSTER .filter(kind => causeDisposition(causeForRecoveryKind(kind)) === "resend-is-futile"); - expect(futile).toEqual([]); + expect(futile).toEqual(["oauth-account-402"]); }); }); diff --git a/tests/responses/responses-account-label.test.ts b/tests/responses/responses-account-label.test.ts index c127c0ae2c0..a88745c707c 100644 --- a/tests/responses/responses-account-label.test.ts +++ b/tests/responses/responses-account-label.test.ts @@ -11,7 +11,8 @@ import { clearThreadAccountMap, getCodexUpstreamHealth, } from "../../src/codex/routing"; -import type { RequestLogContext } from "../../src/server/request-log"; +import { addFinalRequestLog, type RequestLogContext } from "../../src/server/request-log"; +import { createRequestMetricsOwner } from "../../src/server/request-metrics"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -336,6 +337,37 @@ describe("Responses account usage attribution", () => { }); }); + test("a pre-stream 402 move is recorded and counted as a quota recovery, not a rate limit", async () => { + await withPoolHome(async () => { + takeSpendHome(); + const config = poolConfig(["pool-a", "pool-b"]); + for (const id of ["pool-a", "pool-b"]) { + savePoolCredential(id); + updateAccountQuota(id, id === "pool-a" ? 10 : 20); + } + let sends = 0; + globalThis.fetch = (async () => ++sends === 1 + ? Response.json({ error: { message: "payment required" } }, { status: 402 }) + : completedResponse("pool-b-response")) as typeof fetch; + + const metrics = createRequestMetricsOwner(123); + const logCtx: RequestLogContext = { model: "", provider: "", inboundProtocol: "responses", requestMetricsRecorder: metrics }; + const response = await handleResponses(request(), config, logCtx, {}); + expect(response.status).toBe(200); + expect(logCtx.attempts?.map(attempt => [attempt.status, attempt.recoveryKinds])).toEqual([ + [402, []], + [0, ["oauth-account-402"]], + ]); + + addFinalRequestLog("codex-402-move", Date.now() - 1_000, logCtx, 200, { + terminalStatus: "completed", closeReason: "terminal", + }, () => {}); + const recoveries = metrics.snapshot().split("\n").filter(line => line.startsWith("opencodex_recoveries_total{")); + expect(recoveries.find(line => line.includes('recovery="quota"'))?.endsWith(" 1")).toBe(true); + expect(recoveries.find(line => line.includes('recovery="rate_limit"'))?.endsWith(" 0") ?? true).toBe(true); + }); + }); + test("a quota message wrapped in HTTP 502 cools the account and retries an alternate", async () => { await withPoolHome(async () => { takeSpendHome();