Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions gui/src/i18n/de.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1105,6 +1105,7 @@ export const de: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API-Schlüssel erneut authentifiziert",
"logs.detail.attempt.recovery.oauthAccount403": "Kontovalidierung erforderlich (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Konto rate-limitiert (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Kontingent des Kontos erschöpft (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Veralteter verschlüsselter Zustand verworfen",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning-Aufwand reduziert",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Schnellmodus abgelehnt, erneut mit Standardtempo versucht",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/en.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1163,6 +1163,7 @@ export const en = {
"logs.detail.attempt.recovery.key401": "API key re-authentication",
"logs.detail.attempt.recovery.oauthAccount403": "Account validation required (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Account rate-limited (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Account quota exhausted (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Stale encrypted state dropped",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Reasoning effort downgraded",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Fast mode refused, retried at standard speed",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/fr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1139,6 +1139,7 @@ export const fr: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Ré-authentification de la clé API",
"logs.detail.attempt.recovery.oauthAccount403": "Validation du compte requise (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Compte limité (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Quota du compte épuisé (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "État chiffré obsolète abandonné",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Effort de raisonnement réduit",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Mode rapide refusé, nouvel essai à vitesse standard",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ja.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1046,6 +1046,7 @@ export const ja: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API キーの再認証",
"logs.detail.attempt.recovery.oauthAccount403": "アカウントの検証が必要 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "アカウントのレート制限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "アカウントのクォータ枯渇 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "古い暗号化状態を破棄",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "推論強度を下げて再試行",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "高速モードが拒否され、標準速度で再試行",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ko.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1144,6 +1144,7 @@ export const ko: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 키 재인증",
"logs.detail.attempt.recovery.oauthAccount403": "계정 확인 필요 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "계정 속도 제한 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "계정 할당량 소진 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "만료된 암호화 상태 제거",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "추론 강도 하향",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "빠른 모드 거부, 표준 속도로 재시도",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/pt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1158,6 +1158,7 @@ export const pt: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Reautenticação da chave de API",
"logs.detail.attempt.recovery.oauthAccount403": "Validação da conta necessária (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Conta com limite de taxa (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Cota da conta esgotada (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Estado criptografado obsoleto descartado",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Esforço de raciocínio reduzido",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Modo rápido recusado, repetido em velocidade padrão",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/ru.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,7 @@ export const ru: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Повторная аутентификация API-ключа",
"logs.detail.attempt.recovery.oauthAccount403": "Требуется проверка аккаунта (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Ограничение частоты для аккаунта (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Квота аккаунта исчерпана (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Устаревшее зашифрованное состояние отброшено",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Уровень рассуждения понижен",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Быстрый режим отклонён, повтор со стандартной скоростью",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/tr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1149,6 +1149,7 @@ export const tr: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API anahtarı yeniden doğrulandı",
"logs.detail.attempt.recovery.oauthAccount403": "Hesap doğrulaması gerekli (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Hesap hız sınırına takıldı (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Hesap kotası tükendi (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Eski şifreli durum bırakıldı",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Akıl yürütme düzeyi düşürüldü",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Hızlı mod reddedildi, standart hızda yeniden denendi",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/vi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,7 @@ export const vi: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "Xác thực lại khóa API",
"logs.detail.attempt.recovery.oauthAccount403": "Cần xác minh tài khoản (403)",
"logs.detail.attempt.recovery.oauthAccount429": "Tài khoản bị giới hạn tần suất (429)",
"logs.detail.attempt.recovery.oauthAccount402": "Tài khoản đã hết hạn mức (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "Đã bỏ trạng thái mã hóa cũ",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "Đã giảm mức suy luận",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "Chế độ nhanh bị từ chối, thử lại ở tốc độ chuẩn",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/zh-TW.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2948,6 +2948,7 @@ export const zhTW: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 金鑰重新驗證",
"logs.detail.attempt.recovery.oauthAccount403": "需要驗證帳號 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "帳號速率受限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "帳號配額已用盡 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "已捨棄過期加密狀態",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理強度",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式遭拒,改以標準速度重試",
Expand Down
1 change: 1 addition & 0 deletions gui/src/i18n/zh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1117,6 +1117,7 @@ export const zh: Record<TKey, string> = {
"logs.detail.attempt.recovery.key401": "API 密钥重新认证",
"logs.detail.attempt.recovery.oauthAccount403": "需要验证账号 (403)",
"logs.detail.attempt.recovery.oauthAccount429": "账号速率受限 (429)",
"logs.detail.attempt.recovery.oauthAccount402": "账号配额已用尽 (402)",
"logs.detail.attempt.recovery.opaqueBlobRejection": "已丢弃过期加密状态",
"logs.detail.attempt.recovery.reasoningEffortDowngrade": "已降低推理强度",
"logs.detail.attempt.recovery.anthropicFastDowngrade": "快速模式被拒,按标准速度重试",
Expand Down
1 change: 1 addition & 0 deletions gui/src/pages/Logs.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ const RECOVERY_KIND_KEYS = {
"rate-limit-429": "logs.detail.attempt.recovery.rateLimit429",
"anthropic-oauth-429": "logs.detail.attempt.recovery.anthropicOauth429",
"oauth-account-429": "logs.detail.attempt.recovery.oauthAccount429",
"oauth-account-402": "logs.detail.attempt.recovery.oauthAccount402",
"image-413": "logs.detail.attempt.recovery.image413",
"empty-completion": "logs.detail.attempt.recovery.emptyCompletion",
"console-go-upload-retry": "logs.detail.attempt.recovery.consoleGoUpload",
Expand Down
2 changes: 2 additions & 0 deletions src/lib/request-failure-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ const RECOVERY_KIND_CAUSE = {
"rate-limit-429": "rate-limit",
"anthropic-oauth-429": "rate-limit",
"oauth-account-429": "rate-limit",
// A 402 is the account's quota, not a transient limit; only another account can serve it.
"oauth-account-402": "quota-exhausted",
"image-413": "payload-too-large",
// The gateway rejects a body it accepts seconds later and the replay is byte-identical, so
// nothing about the payload was wrong; the origin declined to take it at that moment.
Expand Down
1 change: 1 addition & 0 deletions src/routing/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ const COOLDOWN_RECOVERY_KINDS = new Set([
"oauth-account-403",
"anthropic-oauth-429",
"oauth-account-429",
"oauth-account-402",
]);

function percentile(sorted: number[], p: number): number | undefined {
Expand Down
10 changes: 9 additions & 1 deletion src/server/responses/core-codex-account.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import {
type CodexWsPreOutputRefusalHold,
} from "./codex-ws-wire";
import type { RequestLogContext } from "../request-log";
import type { AttemptRecoveryKind } from "../../usage/log";
import type { DataPlaneAdmission } from "../auth-cors";
import type { InboundWire } from "../../providers/registry";
import type { BunRuntimeGateInput } from "./ws-upstream";
Expand Down Expand Up @@ -971,6 +972,10 @@ export async function retryCodexPoolOnAlternateAccount(
// What this diff does fix is that the rungs are now CHARGED instead of free.
const maxRetrySends = retrySameConfirmedAccount ? 7 : 1;
let retrySendCount = 0;
// The gated-model 400 has no roster kind; it stays unattributed as before.
const moveRecovery: AttemptRecoveryKind | undefined = outcomeStatus === 429 ? "oauth-account-429"
: outcomeStatus === 402 ? "oauth-account-402"
: isTransientUpstreamStatus(outcomeStatus) ? "transient-5xx" : undefined;
let upstreamResponse: Response;
try {
while (true) {
Expand Down Expand Up @@ -1005,7 +1010,10 @@ export async function retryCodexPoolOnAlternateAccount(
// The move is a physical send like any other, so the root workflow is charged too.
chargeWorkflowSends(args.options.workflowRootId, 1);
}
noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate);
// A move seals the first account's attempt here. Give it the status that account actually
// answered, not the 502 the split infers, and name why the second send exists.
if (logCtx.activeAttempt && logCtx.activeAttempt.status < 100) logCtx.activeAttempt.status = firstResponse.status;
noteProviderAttemptSend(logCtx, route.providerName, route.provider, passthroughEstimate, moveRecovery);
try {
upstreamResponse = await fetchWithHeaderTimeout(
request.url,
Expand Down
1 change: 1 addition & 0 deletions src/usage/telemetry-contract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export const ATTEMPT_RECOVERY_KIND_ROSTER = Object.freeze([
"rate-limit-429",
"anthropic-oauth-429",
"oauth-account-429",
"oauth-account-402",
"image-413",
"console-go-upload-retry",
"opaque-blob-rejection",
Expand Down
4 changes: 3 additions & 1 deletion tests/lib/failure-stage-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,10 @@ describe("recovery kinds speak the shared dictionary", () => {
* the durable log were forbidden and unfunded.
*/
test("no recorded recovery classifies as a resend that cannot help", () => {
// Except the Codex 402 account move: re-asking the exhausted account is futile, and the move
// to another account is a route decision this table explicitly does not make.
const futile = ATTEMPT_RECOVERY_KIND_ROSTER
.filter(kind => causeDisposition(causeForRecoveryKind(kind)) === "resend-is-futile");
expect(futile).toEqual([]);
expect(futile).toEqual(["oauth-account-402"]);
});
});
46 changes: 45 additions & 1 deletion tests/responses/responses-account-label.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import {
clearThreadAccountMap,
getCodexUpstreamHealth,
} from "../../src/codex/routing";
import type { RequestLogContext } from "../../src/server/request-log";
import { addFinalRequestLog, type RequestLogContext } from "../../src/server/request-log";
import { createRequestMetricsOwner } from "../../src/server/request-metrics";
import { handleResponses } from "../../src/server/responses";
import type { OcxConfig } from "../../src/types";
import { removeTreeWithRetry } from "../helpers/remove-tree";
Expand Down Expand Up @@ -321,6 +322,49 @@ describe("Responses account usage attribution", () => {
expect(bearers).toEqual(["Bearer pool-a-access-token", "Bearer pool-b-access-token"]);
expect(logCtx.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b"));
expect(logCtx.activeAttempt?.accountLogLabel).toBe(fallbackCodexAccountLogLabel("pool-b"));
// The move is a named recovery: the first account's row keeps its real 429 rather than an
// inferred 502, and the second send says why it exists.
expect(logCtx.attempts?.map(attempt => ({
label: attempt.accountLogLabel,
status: attempt.status,
sendCount: attempt.sendCount,
recoveryKinds: attempt.recoveryKinds,
failureCause: attempt.failureCause,
}))).toEqual([
{ label: fallbackCodexAccountLogLabel("pool-a"), status: 429, sendCount: 1, recoveryKinds: [], failureCause: "rate-limit" },
{ label: fallbackCodexAccountLogLabel("pool-b"), status: 0, sendCount: 1, recoveryKinds: ["oauth-account-429"], failureCause: undefined },
]);
});
});

test("a pre-stream 402 move is recorded and counted as a quota recovery, not a rate limit", async () => {
await withPoolHome(async () => {
takeSpendHome();
const config = poolConfig(["pool-a", "pool-b"]);
for (const id of ["pool-a", "pool-b"]) {
savePoolCredential(id);
updateAccountQuota(id, id === "pool-a" ? 10 : 20);
}
let sends = 0;
globalThis.fetch = (async () => ++sends === 1
? Response.json({ error: { message: "payment required" } }, { status: 402 })
: completedResponse("pool-b-response")) as typeof fetch;

const metrics = createRequestMetricsOwner(123);
const logCtx: RequestLogContext = { model: "", provider: "", inboundProtocol: "responses", requestMetricsRecorder: metrics };
const response = await handleResponses(request(), config, logCtx, {});
expect(response.status).toBe(200);
expect(logCtx.attempts?.map(attempt => [attempt.status, attempt.recoveryKinds])).toEqual([
[402, []],
[0, ["oauth-account-402"]],
]);

addFinalRequestLog("codex-402-move", Date.now() - 1_000, logCtx, 200, {
terminalStatus: "completed", closeReason: "terminal",
}, () => {});
const recoveries = metrics.snapshot().split("\n").filter(line => line.startsWith("opencodex_recoveries_total{"));
expect(recoveries.find(line => line.includes('recovery="quota"'))?.endsWith(" 1")).toBe(true);
expect(recoveries.find(line => line.includes('recovery="rate_limit"'))?.endsWith(" 0") ?? true).toBe(true);
});
});

Expand Down
Loading