From 23a9b671e165fb5544a52cb775cbb0319c329f51 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 4 Oct 2026 00:40:07 +0900 Subject: [PATCH 1/5] fix(codex): preserve stored-main ownership and scoped refresh refusal Carry the credential-provenance and refresh work from #6507 (610824401a488515f6e451d7b926b928ee9d005a and 7ffd1a856957c320d139ff262b4adaf4dd3da07b). Bind native refusal to the physical profile and grant, preserve caller-owned credentials across preview and retry, and retain only fixed refresh diagnostics. Keep terminal-probe attribution from the parent #6515 correction. Co-authored-by: Vadym O Co-authored-by: Claude Opus 5.5 (1M context) --- .../docs/reference/configuration/server.md | 7 + scripts/test-layout/layout.json | 1 + src/codex/account-runtime-state.ts | 21 + src/codex/account-store.ts | 58 +-- src/codex/account-usability.ts | 12 +- src/codex/auth-api/account-list.ts | 136 ++++--- src/codex/auth-api/pool-mode-gate.ts | 12 +- src/codex/auth-context.ts | 15 +- src/codex/chatgpt-refresh-failure.ts | 56 +++ src/codex/main-account.ts | 129 +++++- src/server/responses/codex-auth-error.ts | 12 +- src/server/responses/core-auth.ts | 83 ++-- src/server/responses/core-codex-account.ts | 8 +- src/server/responses/request-prepare.ts | 2 + structure/providers/openai-accounts.md | 21 + ...count-store-refresh-classification.test.ts | 49 ++- .../codex-account-unusable-reason.test.ts | 21 +- .../codex-main-grant-refusal.test.ts | 319 +++++++++++++++ .../main-account-hard-lock-recovery.test.ts | 5 +- tests/fixtures/test-layout-expected.json | 1 + .../responses-native-main-refresh.test.ts | 384 +++++++++++++++++- .../router-blocked-cross-provider.test.ts | 4 +- 22 files changed, 1177 insertions(+), 179 deletions(-) create mode 100644 src/codex/chatgpt-refresh-failure.ts create mode 100644 tests/codex-integration/codex-main-grant-refusal.test.ts diff --git a/docs-site/src/content/docs/reference/configuration/server.md b/docs-site/src/content/docs/reference/configuration/server.md index 53f8344e56a..527c9ce6c10 100644 --- a/docs-site/src/content/docs/reference/configuration/server.md +++ b/docs-site/src/content/docs/reference/configuration/server.md @@ -167,6 +167,13 @@ named a reason that only a new sign-in fixes (`token_invalidated`, `invalid_refr `invalid_workspace_selected`). A `token_invalidated` response indicates a revoked session; the row then shows `needsReauth: true` with the last-known plan. +If the token endpoint rejects the stored main refresh grant, automatic requests stop retrying +that same grant even if a usage refresh succeeds. Sign in again with `codex login` to replace +the credential. The process keeps up to 64 profile-and-grant refusal records; restarting the +proxy or evicting an old record permits a fresh check. Rate-limit and server failures remain +retryable and do not retire the grant. Credentials supplied by the caller remain caller-owned. + + ### Which proxy path is used? The running proxy service fetches quota. It uses its own environment, not the diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 6a7d26dec4e..d4b0cd7ebbd 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -14,6 +14,7 @@ "zed-retained-budget.test.ts": "providers", "jev-decision-scope.test.ts": "server", "codex-main-token-invalidated.test.ts": "codex-integration", + "codex-main-grant-refusal.test.ts": "codex-integration", "codex-shim-path-readiness.test.ts": "codex-integration", "codex-shim-fnm.test.ts": "codex-integration", "claude-intercept-cleanup.test.ts": "claude-integration", diff --git a/src/codex/account-runtime-state.ts b/src/codex/account-runtime-state.ts index 35f3c9cca53..e30bb507c6e 100644 --- a/src/codex/account-runtime-state.ts +++ b/src/codex/account-runtime-state.ts @@ -18,6 +18,27 @@ const reauthAccounts = new Map(); let lastReconciledGeneration = 0; let liveAccountIds = new Set(); +// Refusal evidence is independent of the ordinary quarantine flag. Profile switches make a key +// inapplicable, not forgotten; the oldest of at most 64 records is evicted to bound process state. +const refusedMainRefreshGrants = new Set(); +const MAX_REFUSED_MAIN_REFRESH_GRANTS = 64; + +export function markMainRefreshGrantRejected(key: string): void { + refusedMainRefreshGrants.add(key); + if (refusedMainRefreshGrants.size > MAX_REFUSED_MAIN_REFRESH_GRANTS) { + refusedMainRefreshGrants.delete(refusedMainRefreshGrants.values().next().value!); + } +} + +export function isMainRefreshGrantRejected(key: string): boolean { + return refusedMainRefreshGrants.has(key); +} + +export function clearMainRefreshGrantRejection(key: string): void { + refusedMainRefreshGrants.delete(key); +} + + export function markAccountNeedsReauth( id: string, writerGeneration = captureConfigGeneration(), diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index 8a019d77005..6fceeee8d8f 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -1,3 +1,6 @@ +import { classifyChatgptRefreshFailure, noteChatgptRefreshFailure, type ChatgptRefreshDiagnosticCode } from "./chatgpt-refresh-failure"; +import { readBoundedResponseBody } from "../lib/bounded-body"; +export { classifyChatgptRefreshFailure } from "./chatgpt-refresh-failure"; import { readOrcaAuthSource } from "./orca-auth-source"; import { createHash } from "node:crypto"; import { closeSync, existsSync, fstatSync, readFileSync, mkdirSync, openSync, statSync, unlinkSync, writeFileSync } from "node:fs"; @@ -526,7 +529,8 @@ const CHATGPT_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"; export class TokenRefreshError extends Error { reason: "expired" | "revoked" | "unknown"; - constructor(reason: "expired" | "revoked" | "unknown", message: string) { + constructor(reason: "expired" | "revoked" | "unknown", message: string, + readonly status?: number, readonly code?: ChatgptRefreshDiagnosticCode) { super(message); this.name = "TokenRefreshError"; this.reason = reason; @@ -1322,52 +1326,12 @@ async function resolveCodexToken( signal, }); if (!res.ok) { - const errText = await res.text().catch(() => ""); - let errDesc: string; - let errCodeExact: string | undefined; - try { - const parsed = JSON.parse(errText) as { - error?: string | { code?: string; message?: string }; - error_description?: string; - }; - if (typeof parsed.error === "string") { - errCodeExact = parsed.error.trim(); - errDesc = [parsed.error, parsed.error_description].filter(Boolean).join(": "); - } else if (parsed.error && typeof parsed.error === "object") { - errCodeExact = typeof parsed.error.code === "string" ? parsed.error.code.trim() : undefined; - errDesc = [parsed.error.code, parsed.error.message, parsed.error_description].filter(Boolean).join(": "); - } else { - errDesc = parsed.error_description || `HTTP ${res.status}`; - } - if (!errDesc) errDesc = `HTTP ${res.status}`; - } catch { errDesc = `HTTP ${res.status}`; } - // `invalid_grant` is the standard OAuth code for a refresh token that is no longer - // usable, and upstream sends it bare with no description. Without it here the dead - // grant is classified "unknown", which callers treat as transient — so the account - // is never retired and every request repeats the same doomed refresh (#2887). - // - // Matched on the exact `error` CODE, not anywhere in the combined text: a transient - // `server_error` whose description happens to mention invalid_grant would otherwise - // retire a healthy account, which is the failure this whole change exists to remove. - // - // That rule binds the DESCRIPTION words too. "invalidated", "revoked" and "expired" read - // as terminal prose, but upstream puts arbitrary text there: a `server_error` whose - // description says "token was revoked" or "session expired" is still a 5xx blip, and - // retiring the account on it is exactly the false quarantine #2887 exists to prevent. - // So a body that carries a structured code is classified by that code ALONE. The - // substring fallback survives only where there is no structured code to read at all -- - // a description-only body, or one this parser could not decode -- because there the - // prose is the only signal upstream gave us. - const structuredCode = errCodeExact ? errCodeExact : undefined; - const proseIsOnlySignal = structuredCode === undefined; - const reason = structuredCode === "invalid_grant" - || structuredCode === "refresh_token_invalidated" - || (proseIsOnlySignal - && (errDesc.includes("invalidated") || errDesc.includes("revoked"))) ? "revoked" as const - : structuredCode === "refresh_token_expired" - || (proseIsOnlySignal && errDesc.includes("expired")) ? "expired" as const - : "unknown" as const; - throw new TokenRefreshError(reason, `Codex token refresh failed (${reason}); reauthenticate the account.`); + const body = await readBoundedResponseBody(res, { signal, maxBytes: 16_384, fatalUtf8: true }) + .catch(() => undefined); + const failure = classifyChatgptRefreshFailure(res.status, body?.displaySafe ? body.text : ""); + const { reason } = failure; + noteChatgptRefreshFailure("pool", res.status, failure); + throw new TokenRefreshError(reason, `Codex token refresh failed (${reason}); reauthenticate the account.`, res.status, failure.code); } const data = (await res.json()) as { access_token: string; refresh_token?: string; expires_in: number }; // Guard against a missing/non-finite/negative expires_in (malformed upstream diff --git a/src/codex/account-usability.ts b/src/codex/account-usability.ts index 51c076f6db2..9ca84db8b14 100644 --- a/src/codex/account-usability.ts +++ b/src/codex/account-usability.ts @@ -4,6 +4,7 @@ import { MAIN_CODEX_ACCOUNT_ID, hasMainAccountRefreshGrant, isMainAccountCredentialUsable, + isMainAccountRefreshGrantRejected, isMainAccountTokenLive, } from "./main-account"; import { hasLegacyMainCodexPoolAccount, isSelectableCodexPoolAccount } from "./account-id"; @@ -88,11 +89,14 @@ export function codexAccountUnusableReason( // A legacy pool row with the sentinel makes an active `__main__` ambiguous. // Fail closed until the authenticated compatibility-delete path removes it. if (hasLegacyMainCodexPoolAccount(config.codexAccounts)) return "legacy_pool_sentinel"; - if (isAccountNeedsReauth(accountId) && !hasMainAccountRefreshGrant()) return "needs_reauth"; - // A selection-only caller owns the recovery/drain fence and will reject main - // before reservation or token materialization. Treat cached main as a routing - // candidate without touching the credential file so affinity is not rebound. + // These scopes cannot consume physical stored-main state, even indirectly through an + // ordinary quarantine plus hasMainAccountRefreshGrant(). Caller liveness is request-owned. if (options.nativeMainSelectionOnly) return undefined; + if (options.requestOwnedMainCredential) { + return options.isMainAccountTokenLive?.() === false ? "main_credential_unavailable" : undefined; + } + if (isMainAccountRefreshGrantRejected()) return "needs_reauth"; + if (isAccountNeedsReauth(accountId) && !hasMainAccountRefreshGrant()) return "needs_reauth"; // Main account: a refresh grant is enough to route; materialization refreshes before I/O. const mainLive = options.isMainAccountTokenLive ? options.isMainAccountTokenLive() diff --git a/src/codex/auth-api/account-list.ts b/src/codex/auth-api/account-list.ts index 79b472574df..9dfefd9975c 100644 --- a/src/codex/auth-api/account-list.ts +++ b/src/codex/auth-api/account-list.ts @@ -15,7 +15,7 @@ import { getCodexAccountAutoSwitchThresholdOverride } from "../account-auto-swit import { clearThreadAccountMapForAccount, isCodexAccountPlanExcluded, reconcileCodexActiveAfterExclusion } from "../routing"; import { codexPlanValue, isThirtyDayOnlyCodexPlan } from "../plan"; import { isAccountNeedsReauth, markAccountNeedsReauth } from "../account-runtime-state"; -import { getValidMainAccountToken, MainAccountTokenRefreshError, MAIN_CODEX_ACCOUNT_ID } from "../main-account"; +import { getValidMainAccountToken, isMainAccountRefreshGrantRejected, MAIN_CODEX_ACCOUNT_ID } from "../main-account"; import { captureConfigGeneration } from "../../lib/state-store-sweeper"; import { captureMainAccountIdentityGeneration, getMainAccountCredentialPresence, getMainAccountInfoCache, getObservedMainQuotaIdentityKey, isMainAccountIdentityGenerationLive } from "../main-account-cache"; import type { CodexQuotaRefreshOutcome } from "../quota-refresh-outcome"; @@ -342,62 +342,76 @@ export async function listCodexAuthAccountsSnapshot( maskEmails, )]; }); - const fetchedMainGeneration = mainResult.identityGeneration ?? captureMainAccountIdentityGeneration(); - const mainSnapshotLive = isMainAccountIdentityGenerationLive(fetchedMainGeneration); - // An ordinary same-account return can be parsed after its credential was replaced. - // The card and hard-lock status must describe the same published quota snapshot. - const mainInfo = mainSnapshotLive - ? mainResult.infoUnpublished ? getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO : mainResult.info - : EMPTY_MAIN_ACCOUNT_INFO; - const hasMainCredential = mainSnapshotLive && mainResult.credentialChecked - ? mainResult.hasCredential - : getMainAccountCredentialPresence() ?? false; - const mainMissingCredential = mainSnapshotLive && mainResult.credentialChecked && !hasMainCredential; - const liveQuotaRefresh = mainSnapshotLive && mainResult.quotaRefreshGeneration !== undefined - && isMainAccountIdentityGenerationLive(mainResult.quotaRefreshGeneration) - ? mainResult.quotaRefresh : undefined; - const mainNeedsReauth = mainMissingCredential || isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); - const mainHealth = projectCodexAccountHealth({ - accountId: MAIN_CODEX_ACCOUNT_ID, - needsReauth: mainNeedsReauth, - }); - // The main row carries the same attribution as a pool row. Reaching this point without - // A diagnostic status is not causal evidence: a live bare 401 can be transient while an - // earlier refresh failure still owns the quarantine. Only a current terminal probe attributes it. - const mainReauthReason: CodexAccountReauthReason | undefined = mainMissingCredential - ? "missing_credential" - : mainNeedsReauth - ? mainResult.terminalAuthFailure === true && liveQuotaRefresh?.status === "http_error" - ? "unauthorized" : "refresh_failed" - : mainHealth.status === "reauth_required" ? mainHealth.reason : undefined; - const main: CodexAuthAccountDto = { - id: MAIN_CODEX_ACCOUNT_ID, - email: projectEmail(mainInfo.email, maskEmails) ?? "Codex App login", - plan: mainInfo.plan, - ...(liveQuotaRefresh ? { quotaRefresh: liveQuotaRefresh } : {}), - logLabel: "main", - isMain: true, - paused: isCodexAccountPaused(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), - mainAccountHardLock: { ...getMainAccountHardLockStatus(runtimeConfig), - externalUsage: getMainAccountExternalUsageWarning(getObservedMainQuotaIdentityKey()) }, - priority: getCodexAccountPriority(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), - autoSwitchThresholdOverride: getCodexAccountAutoSwitchThresholdOverride(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), - creditsAfterLimit: codexAccountUsesCreditsAfterLimit(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), - hasCredential: hasMainCredential, - needsReauth: mainNeedsReauth, - ...(mainReauthReason !== undefined ? { reauthReason: mainReauthReason } : {}), - quota: mainInfo.quota - ? quotaForPlan(mainQuotaWithCarriedResetCredits(mainInfo.quota), mainInfo.plan) - : null, - ...codexCreditsDtoField(runtimeConfig, MAIN_CODEX_ACCOUNT_ID, getMainChatgptAccountId()), - ...oauthAccountHealthFields("codex", MAIN_CODEX_ACCOUNT_ID, mainHealth), - }; - return { - accounts: [main, ...withQuota], - mainIdentityGeneration: mainSnapshotLive - ? fetchedMainGeneration - : captureMainAccountIdentityGeneration(), + const projectSnapshot = (mainRefreshRefused = false): CodexAuthAccountsSnapshot => { + const fetchedMainGeneration = mainResult.identityGeneration ?? captureMainAccountIdentityGeneration(); + const mainSnapshotLive = isMainAccountIdentityGenerationLive(fetchedMainGeneration); + // An ordinary same-account return can be parsed after its credential was replaced. + // The card and hard-lock status must describe the same published quota snapshot. + const mainInfo = mainSnapshotLive + ? mainResult.infoUnpublished ? getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO : mainResult.info + : EMPTY_MAIN_ACCOUNT_INFO; + const hasMainCredential = mainSnapshotLive && mainResult.credentialChecked + ? mainResult.hasCredential + : getMainAccountCredentialPresence() ?? false; + const mainMissingCredential = mainSnapshotLive && mainResult.credentialChecked && !hasMainCredential; + const liveQuotaRefresh = mainSnapshotLive && mainResult.quotaRefreshGeneration !== undefined + && isMainAccountIdentityGenerationLive(mainResult.quotaRefreshGeneration) + ? mainResult.quotaRefresh : undefined; + const mainNeedsReauth = mainMissingCredential || mainRefreshRefused || isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + const mainHealth = projectCodexAccountHealth({ + accountId: MAIN_CODEX_ACCOUNT_ID, + needsReauth: mainNeedsReauth, + }); + // The main row carries the same attribution as a pool row. Reaching this point without + // Only a current terminal usage probe can attribute the reauth cause. A transient status + // never replaces a refresh failure, including a separately recorded refused grant. + const mainReauthReason: CodexAccountReauthReason | undefined = mainMissingCredential + ? "missing_credential" + : mainNeedsReauth + ? mainResult.terminalAuthFailure === true && liveQuotaRefresh?.status === "http_error" + ? "unauthorized" : "refresh_failed" + : mainHealth.status === "reauth_required" ? mainHealth.reason : undefined; + const main: CodexAuthAccountDto = { + id: MAIN_CODEX_ACCOUNT_ID, + email: projectEmail(mainInfo.email, maskEmails) ?? "Codex App login", + plan: mainInfo.plan, + ...(liveQuotaRefresh ? { quotaRefresh: liveQuotaRefresh } : {}), + logLabel: "main", + isMain: true, + paused: isCodexAccountPaused(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), + mainAccountHardLock: { ...getMainAccountHardLockStatus(runtimeConfig), + externalUsage: getMainAccountExternalUsageWarning(getObservedMainQuotaIdentityKey()) }, + priority: getCodexAccountPriority(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), + autoSwitchThresholdOverride: getCodexAccountAutoSwitchThresholdOverride(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), + creditsAfterLimit: codexAccountUsesCreditsAfterLimit(runtimeConfig, MAIN_CODEX_ACCOUNT_ID), + hasCredential: hasMainCredential, + needsReauth: mainNeedsReauth, + ...(mainReauthReason !== undefined ? { reauthReason: mainReauthReason } : {}), + quota: mainInfo.quota + ? quotaForPlan(mainQuotaWithCarriedResetCredits(mainInfo.quota), mainInfo.plan) + : null, + ...codexCreditsDtoField(runtimeConfig, MAIN_CODEX_ACCOUNT_ID, getMainChatgptAccountId()), + ...oauthAccountHealthFields("codex", MAIN_CODEX_ACCOUNT_ID, mainHealth), + }; + return { + accounts: [main, ...withQuota], + mainIdentityGeneration: mainSnapshotLive + ? fetchedMainGeneration + : captureMainAccountIdentityGeneration(), + }; }; + // Project the refusal and DTO in the same owned synchronous observation. A same-account grant + // replacement during earlier pool probes must not inherit the previous grant's refusal. + const projectionLease = tryAcquireNativeMainProfileClaim(); + if (!projectionLease) return projectSnapshot(); + try { + return await withNativeMainCredentialClaim(async () => projectSnapshot(isMainAccountRefreshGrantRejected())); + } catch (error) { + if (isNativeMainClaimUnavailable(error)) return projectSnapshot(); + throw error; + } finally { + projectionLease.release(); + } } /** One opted-in account's metadata; reuse the bounded WHAM 401 recovery and generation fence. */ @@ -408,16 +422,10 @@ export async function refreshCodexQuotaForActivation(config: OcxConfig, accountI try { reconcileMainCodexAccountRuntimeState(); if (isAccountNeedsReauth(accountId)) return; - const identityGeneration = captureMainAccountIdentityGeneration(); - const writerGeneration = captureConfigGeneration(); try { - // Refresh may need an exclusive claim; prepare before WHAM takes its shared claim. + // Refresh owns credential-scoped refusal evidence; do not recreate a global quarantine. if (!await getValidMainAccountToken({ preserveReauth: true })) return; - } catch (error) { - if (error instanceof MainAccountTokenRefreshError && error.reason === "reauth" - && isMainAccountIdentityGenerationLive(identityGeneration)) { - markAccountNeedsReauth(accountId, writerGeneration); - } + } catch { return; } if (isAccountNeedsReauth(accountId)) return; diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index 83048e457ce..e3c9393bff7 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -7,8 +7,8 @@ import { getAccountQuota, isCompleteCodexQuotaRecoverySnapshot } from "../quota" import { reconcileMainCodexAccountRuntimeState } from "../account-lifecycle"; import { claimDueCodexQuotaRecoveryProbes, settleCodexQuotaRecoveryProbe } from "../routing"; import { readCodexTokens } from "../auth-collision"; -import { isAccountNeedsReauth, markAccountNeedsReauth } from "../account-runtime-state"; -import { getValidMainAccountToken, MainAccountTokenRefreshError, MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "../main-account"; +import { isAccountNeedsReauth } from "../account-runtime-state"; +import { getValidMainAccountToken, MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "../main-account"; import { captureConfigGeneration, registerStateSweepAfterTick } from "../../lib/state-store-sweeper"; import { observeMainQuotaCredential, getMainQuotaCredentialGeneration, captureMainAccountIdentityGeneration, isMainAccountIdentityGenerationLive } from "../main-account-cache"; import { getMainAccountHardLockStatus } from "../main-account-hard-lock"; @@ -101,18 +101,14 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise if (getMainAccountHardLockStatus(config).state !== "blocked" || isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; const identityGeneration = captureMainAccountIdentityGeneration(); - const writerGeneration = captureConfigGeneration(); try { // Refresh can require an exclusive credential claim: never hold WHAM's shared // claim while obtaining a valid token. The runtime lease spans both operations. const prepared = await getValidMainAccountToken({ preserveReauth: true }); if (!prepared) return; observeMainQuotaCredential(prepared.accessToken, prepared.chatgptAccountId); - } catch (error) { - if (error instanceof MainAccountTokenRefreshError && error.reason === "reauth" - && isMainAccountIdentityGenerationLive(identityGeneration)) { - markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID, writerGeneration); - } + } catch { + // Native refresh retains its own grant-scoped refusal; global quarantine would outlive it. return; } if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index f2355961567..99ef6aeec09 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -17,7 +17,7 @@ import { import { isAccountNeedsReauth, markAccountNeedsReauth } from "./account-runtime-state"; import { ConfigMutationLockError } from "../config"; import { NativeProfileError } from "./native-profile-types"; -import { isCodexAccountUsable } from "./account-usability"; +import { codexAccountUnusableReason, isCodexAccountUsable } from "./account-usability"; import { reconcileMainCodexAccountRuntimeState } from "./account-lifecycle"; import { MAIN_CODEX_ACCOUNT_ID, @@ -379,9 +379,11 @@ export class CodexAuthContextError extends Error { } export class CodexPoolAuthenticationError extends Error { - constructor(message = "OpenAI account pool has no usable account credential") { + readonly quarantinedMain: boolean; + constructor(message = "OpenAI account pool has no usable account credential", options?: { quarantinedMain?: boolean }) { super(message); this.name = "CodexPoolAuthenticationError"; + this.quarantinedMain = options?.quarantinedMain === true; } } @@ -922,7 +924,7 @@ export function shouldMarkAccountNeedsReauthForCodexAuthFailure(cause: unknown): && !(cause instanceof CodexCredentialRefreshBusyError) && !(cause instanceof CodexCredentialRefreshStaleError) && !(cause instanceof MainAuthJsonChangedDuringRefreshError) - && !(cause instanceof MainAccountTokenRefreshError && cause.reason === "transient") + && !(cause instanceof MainAccountTokenRefreshError) && !(cause instanceof NativeProfileError && cause.retryable) && !(cause instanceof DOMException && cause.name === "AbortError") && !(cause instanceof ConfigMutationLockError); @@ -1341,7 +1343,12 @@ export async function resolveCodexAuthContext( : "Codex accounts that support this model are currently unavailable", ); } - throw new CodexPoolAuthenticationError(); + throw new CodexPoolAuthenticationError(undefined, { + quarantinedMain: !nativeMainReadsForbidden + && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID + && !policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID) + && codexAccountUnusableReason(config, MAIN_CODEX_ACCOUNT_ID, selectionOptions) === "needs_reauth", + }); } accountId = selected; if (accountId === MAIN_CODEX_ACCOUNT_ID) assertMainAccountPolicy(policy); diff --git a/src/codex/chatgpt-refresh-failure.ts b/src/codex/chatgpt-refresh-failure.ts new file mode 100644 index 00000000000..40499a4db79 --- /dev/null +++ b/src/codex/chatgpt-refresh-failure.ts @@ -0,0 +1,56 @@ +/** Closed diagnostic vocabulary; provider descriptions and unknown codes never escape. */ +const REFRESH_DIAGNOSTIC_CODES = [ + "invalid_grant", "refresh_token_invalidated", "token_invalidated", "refresh_token_reused", + "refresh_token_expired", "invalid_client", "invalid_request", "server_error", + "temporarily_unavailable", "rate_limit_exceeded", "unsupported_grant_type", "invalid_scope", +] as const; +export type ChatgptRefreshDiagnosticCode = typeof REFRESH_DIAGNOSTIC_CODES[number]; +export type ChatgptRefreshFailure = { + reason: "expired" | "revoked" | "unknown"; + code?: ChatgptRefreshDiagnosticCode; +}; + +/** HTTP availability failures cannot establish that a refresh grant was revoked. */ +export function classifyChatgptRefreshFailure(status: number, body: string): ChatgptRefreshFailure { + if (body.length > 16_384) return { reason: "unknown" }; + let payload: unknown; + try { payload = JSON.parse(body); } catch { return { reason: "unknown" }; } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return { reason: "unknown" }; + const row = payload as Record; + let exactCode: string | undefined; + let description = typeof row.error_description === "string" ? row.error_description : ""; + if (typeof row.error === "string") { + exactCode = row.error.trim() || undefined; + } else if (row.error !== undefined) { + if (!row.error || typeof row.error !== "object" || Array.isArray(row.error)) return { reason: "unknown" }; + const error = row.error as Record; + if (error.code !== undefined && typeof error.code !== "string") return { reason: "unknown" }; + exactCode = typeof error.code === "string" ? error.code.trim() || undefined : undefined; + if (typeof error.message === "string") description = `${error.message} ${description}`; + } + const code = (REFRESH_DIAGNOSTIC_CODES as readonly unknown[]).includes(exactCode) + ? exactCode as ChatgptRefreshDiagnosticCode : undefined; + let reason: ChatgptRefreshFailure["reason"] = "unknown"; + if (status === 400 || status === 401 || status === 403) { + if (["invalid_grant", "refresh_token_invalidated", "token_invalidated", "refresh_token_reused"].includes(exactCode ?? "")) { + reason = "revoked"; + } else if (exactCode === "refresh_token_expired") { + reason = "expired"; + } else if (status === 400 && exactCode === undefined) { + // Preserve description-only OAuth 400 compatibility; a named code always wins. + if (description.includes("invalidated") || description.includes("revoked")) reason = "revoked"; + else if (description.includes("expired")) reason = "expired"; + } + } + return { reason, ...(code ? { code } : {}) }; +} + +/** Endpoint verdict only: no credential identity, body, or arbitrary upstream string. */ +export function noteChatgptRefreshFailure( + kind: "pool" | "native main", + status: number, + failure: ChatgptRefreshFailure, +): void { + console.warn(`[codex] ${kind} refresh: ${failure.reason === "unknown" ? "transient" : "reauth"}` + + ` status=${status} code=${failure.code ?? "none"}`); +} diff --git a/src/codex/main-account.ts b/src/codex/main-account.ts index 09a11eaa86a..d62facb3245 100644 --- a/src/codex/main-account.ts +++ b/src/codex/main-account.ts @@ -1,11 +1,15 @@ +import { classifyChatgptRefreshFailure, noteChatgptRefreshFailure } from "./chatgpt-refresh-failure"; +import { readBoundedResponseBody } from "../lib/bounded-body"; import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; import { join } from "node:path"; import { readCodexTokens } from "./auth-collision"; import { + CHATGPT_CLIENT_ID, + CHATGPT_TOKEN_URL, + credsFromToken, decodeJwtPayload, extractAccountId, - refreshChatGPTToken, } from "../oauth/chatgpt"; import type { OAuthCredentials } from "../oauth/types"; import { extractChatgptPlanType } from "./plan"; @@ -17,7 +21,12 @@ import { import { atomicWriteFile, resolveWriteTarget } from "../config/atomic-write"; import { resolveCodexHomeDir } from "./home"; import { assertNotRealCodexHomeUnderTest } from "../lib/test-home-guard"; -import { clearAccountNeedsReauth } from "./account-runtime-state"; +import { + clearAccountNeedsReauth, + isMainRefreshGrantRejected, + markMainRefreshGrantRejected, + clearMainRefreshGrantRejection, +} from "./account-runtime-state"; import { advanceCodexCredentialMutationEpoch } from "./credential-mutation-epoch"; import { withNativeMainExclusiveClaim } from "./native-main-claim"; import { resolveNativeProfileContext } from "./native-profile-store"; @@ -78,6 +87,59 @@ export class MainAccountTokenRefreshError extends Error { } } +/** + * The native-main refresh, posted here rather than through `refreshChatGPTToken` so the token + * endpoint's structured `error` code survives to {@link classifyChatgptRefreshFailure}. + * + * `refreshChatGPTToken` collapses the refusal into one display string, and classifying a grant + * from that string is what let `refresh_token_reused` pass as transient -- the grant never + * retired, every request re-earning its own 401 -- while a 5xx whose description merely read + * "session expired" passed as proof and would have retired a live grant. + * + * Same endpoint, client id and grant type as every other ChatGPT refresh, and the success path + * is `credsFromToken` itself, so only the failure branch differs: it classifies by code, exactly + * as the stored-pool refresh does. + */ +async function refreshNativeMainGrant( + refreshToken: string, + options: { signal: AbortSignal }, +): Promise { + const resp = await fetch(CHATGPT_TOKEN_URL, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "refresh_token", + client_id: CHATGPT_CLIENT_ID, + refresh_token: refreshToken, + }).toString(), + signal: options.signal, + }); + if (!resp.ok) { + const body = await readBoundedResponseBody(resp, { signal: options.signal, maxBytes: 16_384, fatalUtf8: true }) + .catch(() => undefined); + const failure = classifyChatgptRefreshFailure(resp.status, body?.displaySafe ? body.text : ""); + noteChatgptRefreshFailure("native main", resp.status, failure); + throw new MainAccountTokenRefreshError(failure.reason === "unknown" ? "transient" : "reauth"); + } + noteNativeMainRefresh("ok", resp.status); + return credsFromToken((await resp.json()) as Record); +} + +/** + * One line per native-main refresh verdict, so a dead grant is diagnosable from service.log. + * + * A revoked session is invisible from the outside: the grant is refused, nothing is written, and + * the request fails with a status that looks like every other 401. Carries the token endpoint's + * HTTP status and its structured `error` code and nothing else -- never a token, never a body. + */ +function noteNativeMainRefresh(outcome: string, status?: number, code?: string): void { + console.warn( + `[codex] native main refresh: ${outcome}` + + (status === undefined ? "" : ` status=${status}`) + + ` code=${code && /^[A-Za-z0-9._-]{1,64}$/.test(code) ? code : "none"}`, + ); +} + function nonEmptyString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value : undefined; } @@ -147,13 +209,30 @@ function mainAccessTokenFresh(accessToken: string | undefined, now: number, skew } /** A refresh grant makes native main routeable even when the current access token is expired. */ +function mainRefreshGrantKey(current: MainAuthJsonCredential): string | undefined { + return current.refreshToken + ? `${current.path}:${refreshGrantFingerprintForToken(current.refreshToken)}` : undefined; +} + +function credentialRefreshGrantRejected(current: MainAuthJsonCredential | null): boolean { + const key = current && mainRefreshGrantKey(current); + return !!key && isMainRefreshGrantRejected(key); +} + +/** Current owned profile only; callers must hold the native read/selection permission. */ +export function isMainAccountRefreshGrantRejected(): boolean { + return credentialRefreshGrantRejected(readMainAuthJsonCredential()); +} + export function isMainAccountCredentialUsable(now = Date.now()): boolean { const current = readMainAuthJsonCredential(); - return !!current?.refreshToken || mainAccessTokenFresh(current?.accessToken, now, 0); + return !credentialRefreshGrantRejected(current) + && (!!current?.refreshToken || mainAccessTokenFresh(current?.accessToken, now, 0)); } export function hasMainAccountRefreshGrant(): boolean { - return !!readMainAuthJsonCredential()?.refreshToken; + const current = readMainAuthJsonCredential(); + return !!current?.refreshToken && !credentialRefreshGrantRejected(current); } function assertMainAuthJsonSnapshotUnchanged(expected: MainAuthJsonCredential): void { @@ -342,14 +421,26 @@ async function resolveMainAccountToken( dependencies: NativeMainRefreshDependencies = {}, rejectedAccessToken?: string, ): Promise<{ accessToken: string; chatgptAccountId: string } | null> { + if (dependencies.signal?.aborted) throw dependencies.signal.reason; const initial = readMainAuthJsonCredential(); - if (!initial) return null; + // Every skip below ends a forced refresh WITHOUT asking the token endpoint anything, which from + // the outside is indistinguishable from a refusal: same failed request, no write, no verdict. + // A forced refresh is the only caller that passes `rejectedAccessToken`, so only it reports. + if (!initial) { + if (rejectedAccessToken !== undefined) noteNativeMainRefresh("skipped-no-credential"); + return null; + } + if (credentialRefreshGrantRejected(initial)) throw new MainAccountTokenRefreshError("reauth"); const now = Date.now(); if (initial.accessToken !== rejectedAccessToken && mainAccessTokenFresh(initial.accessToken, now, MAIN_TOKEN_REFRESH_SKEW_MS)) { + // The stored credential is no longer the one upstream rejected: somebody else replaced it + // while this request was in flight, so there is nothing here to refresh or to retire. + if (rejectedAccessToken !== undefined) noteNativeMainRefresh("skipped-credential-replaced"); return { accessToken: initial.accessToken!, chatgptAccountId: initial.chatgptAccountId }; } if (!initial.refreshToken) { + if (rejectedAccessToken !== undefined) noteNativeMainRefresh("skipped-no-refresh-grant"); return initial.accessToken !== rejectedAccessToken && mainAccessTokenFresh(initial.accessToken, now, 0) ? { accessToken: initial.accessToken!, chatgptAccountId: initial.chatgptAccountId } @@ -380,6 +471,7 @@ async function resolveMainAccountToken( () => withCodexRefreshFileLock(lockKey, signal, async () => { const locked = readMainAuthJsonCredential(); if (!locked) throw new MainAuthJsonChangedDuringRefreshError(); + if (credentialRefreshGrantRejected(locked)) throw new MainAccountTokenRefreshError("reauth"); if (!locked.refreshToken || refreshGrantFingerprintForToken(locked.refreshToken) !== lockKey) { if (locked.accessToken !== rejectedAccessToken @@ -392,16 +484,31 @@ async function resolveMainAccountToken( && mainAccessTokenFresh(locked.accessToken, Date.now(), MAIN_TOKEN_REFRESH_SKEW_MS)) { return { accessToken: locked.accessToken!, chatgptAccountId: locked.chatgptAccountId }; } - const refresh = dependencies.refreshToken - ?? ((refreshToken: string, options: { signal: AbortSignal }) => refreshChatGPTToken(refreshToken, options)); + const refresh = dependencies.refreshToken ?? refreshNativeMainGrant; let refreshed: OAuthCredentials; try { refreshed = await refresh(locked.refreshToken, { signal }); } catch (cause) { + // A refusal already classified at the endpoint keeps that verdict. An injected refresh + // threw an ordinary Error and has only its message, so prose still decides there -- + // and only there, where there is no structured code to read. const message = cause instanceof Error ? cause.message.toLowerCase() : ""; - const reason = /invalid_grant|invalidated|revoked|expired/.test(message) - ? "reauth" as const - : "transient" as const; + const reason = cause instanceof MainAccountTokenRefreshError + ? cause.reason + : dependencies.refreshToken && /invalid_grant|invalidated|revoked|expired/.test(message) + ? "reauth" as const + : "transient" as const; + // The shared point every request path reaches a terminal native-main verdict through: + // the forced refresh after an upstream 401, the pre-send refresh of an expired bearer, + // and the compact twin all land here. Recording the quarantine at the callers instead + // left the native main slot alone unrecorded -- `refreshNativeMainForwardAuth` has no + // `quarantine` flag for its caller to act on the way the stored-pool twin does -- so a + // revoked session stayed `needsReauth: false` and every request re-earned its own 401. + if (signal.aborted) throw signal.reason; + if (reason === "reauth") { + assertMainAuthJsonSnapshotUnchanged(locked); + markMainRefreshGrantRejected(mainRefreshGrantKey(locked)!); + } throw new MainAccountTokenRefreshError(reason, { cause }); } // The refresh may resolve after the caller went away (an implementation that does @@ -411,6 +518,8 @@ async function resolveMainAccountToken( // before its own commit above. if (dependencies.signal?.aborted) throw dependencies.signal.reason; const result = persistRefreshedMainAuthJson(locked, refreshed); + // A permitted in-flight success speaks only for its own profile and grant. + clearMainRefreshGrantRejection(mainRefreshGrantKey(locked)!); if (dependencies.preserveReauth !== true) clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); return result; }), diff --git a/src/server/responses/codex-auth-error.ts b/src/server/responses/codex-auth-error.ts index 12ab9a40fa5..303ee664a86 100644 --- a/src/server/responses/codex-auth-error.ts +++ b/src/server/responses/codex-auth-error.ts @@ -30,9 +30,15 @@ export function codexModelAvailabilityErrorResponse(error: CodexModelAvailabilit return formatErrorResponse(400, "invalid_request_error", error.message); } +/** Actionable sign-in guidance without guessing which upstream revocation cause occurred. */ +export const CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE = + "Codex main account needs sign-in. Run `codex login` to sign in again."; + export function nativeMainRefreshFailureResponse(error: unknown): Response { if (error instanceof MainAccountTokenRefreshError && error.reason === "reauth") { - return formatErrorResponse(401, "authentication_error", "Codex main account needs reauthentication"); + // Same sentence the quarantine produces on every later request, so the first refusal and the + // ones after it do not describe one dead credential two different ways. + return formatErrorResponse(401, "authentication_error", CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); } if (error instanceof MainAccountTokenRefreshError || error instanceof MainAuthJsonChangedDuringRefreshError @@ -98,7 +104,9 @@ export function mapCodexAuthContextErrorToResponse( return codexModelAvailabilityErrorResponse(error); } if (error instanceof CodexPoolAuthenticationError || error instanceof CodexDirectAuthenticationError) { - return formatErrorResponse(401, "authentication_error", error.message); + return formatErrorResponse(401, "authentication_error", + error instanceof CodexPoolAuthenticationError && error.quarantinedMain + ? CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE : error.message); } if (error instanceof CodexMainSubstitutionUnavailableError) { return formatErrorResponse( diff --git a/src/server/responses/core-auth.ts b/src/server/responses/core-auth.ts index 3eb902cb3de..32d6f56d115 100644 --- a/src/server/responses/core-auth.ts +++ b/src/server/responses/core-auth.ts @@ -57,6 +57,9 @@ export type ResponsesAuthResolution = | { ok: false; response: Response }; +export type CodexCredentialOwnershipOptions = Pick; + /** * The caller credential the final Codex auth resolution will be given, as far as the ROUTE * decides it: a route change that may cross a credential domain drops the raw caller credential, @@ -67,15 +70,53 @@ export type ResponsesAuthResolution = * an HMAC of exactly this Authorization header. Two copies of this rule would put preview and * final auth in different scopes the first time one of them changed. */ +/** + * The stored native-main credential this process injected itself, when it is the bearer the + * resolution will see. + * + * `handleClaudeMessages` reads `~/.codex/auth.json` and attaches it so OpenAI-backed sidecars stay + * reachable on a translated turn. It is OUR credential, not the caller's -- the ingress says so + * where it sets `nativeCallerAuth` and `callerDirectAuth` to null for exactly this reason -- and + * every reader of the resolution's input headers has to agree about that, or one of them conclud + * the request owns a credential that in fact belongs to the Pool. + */ +export function injectedClaudeMainBearer( + route: Pick, + options: CodexCredentialOwnershipOptions, +): { authorization: string; chatgptAccountId?: string } | undefined { + return options.stripClaudeMainAuthForNoncanonicalForward === true + && isCanonicalOpenAiForwardProvider(route.provider) + ? options.trustedClaudeMainAuth : undefined; +} + +function validatedCallerDirectHeaders( + config: OcxConfig, + route: Pick, + options: CodexCredentialOwnershipOptions, +): Headers | undefined { + if (!options.callerDirectAuth || !isCanonicalOpenAiForwardProvider(route.provider)) return undefined; + const headers = new Headers({ authorization: options.callerDirectAuth.authorization }); + if (options.callerDirectAuth.chatgptAccountId) headers.set("chatgpt-account-id", options.callerDirectAuth.chatgptAccountId); + return captureCallerDirectAuth(headers, config) ? headers : undefined; +} + export function codexRouteCredentialDomainHeaders( req: Request, route: RouteResult, options: HandleResponsesOptions, credentialDomainWasRewritten: boolean, + config: OcxConfig, ): Headers { - const trustedClaudeMainForFinalRoute = options.stripClaudeMainAuthForNoncanonicalForward === true - && isCanonicalOpenAiForwardProvider(route.provider) - ? options.trustedClaudeMainAuth : undefined; + const callerDirect = validatedCallerDirectHeaders(config, route, options); + if (callerDirect) { + const headers = new Headers(req.headers); + headers.set("authorization", callerDirect.get("authorization")!); + const accountId = callerDirect.get("chatgpt-account-id"); + if (accountId) headers.set("chatgpt-account-id", accountId); + else headers.delete("chatgpt-account-id"); + return headers; + } + const trustedClaudeMainForFinalRoute = injectedClaudeMainBearer(route, options); if (trustedClaudeMainForFinalRoute) { const claudeMainHeaders = new Headers(req.headers); claudeMainHeaders.set("authorization", trustedClaudeMainForFinalRoute.authorization); @@ -117,15 +158,25 @@ export function codexRouteCredentialDomainHeaders( export function codexRouteCredentialOwnership( authInputHeaders: Headers, config: OcxConfig, - route: RouteResult, - options: HandleResponsesOptions, + route: Pick, + options: CodexCredentialOwnershipOptions, ): { substituteMainCredential: boolean; requestScopedMainCredential: boolean } { const substituteMainCredential = options.admission?.source === "bearer" && (route.codexAccountMode !== undefined || isCanonicalOpenAiForwardProvider(route.provider)); return { substituteMainCredential, + // A request-scoped main credential is one the CALLER owns: it exists for this request only, + // owns no Pool state, and must never be quarantined or refreshed on the Pool's behalf. The + // stored credential this process injected is the opposite of that in every respect, and + // `hasForwardableCodexBearer` cannot tell them apart -- it sees a Codex JWT with an account + // id either way. Counting ours as the caller's made the native main slot caller-owned on + // every translated Claude turn: `main` instead of `main-pool`, so an upstream 401 on a + // revoked session recorded no outcome, refreshed nothing, and retired nothing, and the next + // request sent the same dead token again. requestScopedMainCredential: route.codexAccountMode !== undefined && !substituteMainCredential + && (validatedCallerDirectHeaders(config, route, options) !== undefined + || injectedClaudeMainBearer(route, options) === undefined) && hasForwardableCodexBearer(authInputHeaders, config), }; } @@ -149,6 +200,7 @@ export async function resolveResponsesCodexAuth( route, options, credentialDomainWasRewritten, + config, ); // A caller-auth transport that is not canonical OpenAI (keyless Cursor) consumes the // caller's Authorization as its own upstream token. Keep that contract only for a clean @@ -171,27 +223,6 @@ export async function resolveResponsesCodexAuth( authInputHeaders = scoped; } } - // The caller's own Direct credential may cross an internal route change only to the - // canonical OpenAI transport, under a predicate deliberately STRICTER than plain - // unchanged-route Direct forwarding: a clean non-proxy bearer whose ChatGPT-domain - // marker is valid, with any explicit account header matching that marker. Unchanged - // routes keep their legacy rules; sidecar enrichment grants no primary authority. - if (options.callerDirectAuth && isCanonicalOpenAiForwardProvider(route.provider)) { - const directHeaders = new Headers({ - authorization: options.callerDirectAuth.authorization, - ...(options.callerDirectAuth.chatgptAccountId - ? { "chatgpt-account-id": options.callerDirectAuth.chatgptAccountId } : {}), - }); - if (captureCallerDirectAuth(directHeaders, config)) { - authInputHeaders = new Headers(authInputHeaders); - authInputHeaders.set("authorization", options.callerDirectAuth.authorization); - if (options.callerDirectAuth.chatgptAccountId) { - authInputHeaders.set("chatgpt-account-id", options.callerDirectAuth.chatgptAccountId); - } else { - authInputHeaders.delete("chatgpt-account-id"); - } - } - } // #1686: a caller that proved admission with a BEARER presented one of our own secrets. // Refusing it here is what made the codex-cli `env_key` contract unusable against Direct. // Admitting it is only safe because the stored main credential is substituted below, so diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 2f2c7413368..6b0540938ef 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -59,7 +59,7 @@ import { import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "../../codex/catalog/native-models"; import { isRequestExecutionBudget } from "../../lib/request-execution-budget"; import type { SingleUseDispatchPermit } from "../../lib/request-execution-budget"; -import { hasForwardableCodexBearer } from "../auth-cors"; +import { codexRouteCredentialOwnership, type CodexCredentialOwnershipOptions } from "./core-auth"; import { bindRouteReasoningReplayScope } from "./core-replay"; import { conversationStateBindingFromAuth, @@ -360,7 +360,7 @@ export interface CodexPoolAccountRetryArgs { route: Pick; parsed: OcxParsedRequest; logCtx: RequestLogContext; - options: { + options: CodexCredentialOwnershipOptions & { admission?: DataPlaneAdmission; codexAuthPolicy?: CodexAuthPolicyConfig; visionDescribeTerminal?: boolean; @@ -644,7 +644,9 @@ export async function retryCodexPoolOnAlternateAccount( admission: options.admission, codexAuthPolicy: options.codexAuthPolicy, modelId: route.modelId, - requestScopedMainCredential: hasForwardableCodexBearer(callerAuthHeaders, config), + requestScopedMainCredential: codexRouteCredentialOwnership(callerAuthHeaders, config, { + provider: route.provider, codexAccountMode: "pool", + }, options).requestScopedMainCredential, beginCodexAccountSelection: codexAccountSelectionForTurn(options.turnAdmissionLease), resolveCodexModelEntitlements: entitlementResolver, }, diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 6dcfdb4d53c..41755eacad8 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -676,6 +676,7 @@ export async function prepareResponsesRequest( route, options, credentialDomainWasRewritten, + config, ); // Does the CALLER own the credential this request will authenticate with? Validated exactly // the way final auth validates it: the route ownership predicate AND the caller-bearer check @@ -962,6 +963,7 @@ export async function prepareResponsesRequest( route, options, credentialDomainWasRewritten, + config, ); const recoveryRequestScopedMainCredential = codexRouteCredentialOwnership( recoveryAuthHeaders, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 537368a7945..46f386eec27 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -201,6 +201,12 @@ sidecar candidate and cannot hide a failed Codex credential with separately bill `src/server/audio-upstream.ts` uses the same selection for standalone transcription. Explicit native Direct auth remains caller-owned; proxy-key-only Direct claims stored main before materialization, replacing both bearer and account identity exclusively from that credential. +Credential OWNERSHIP is decided by provenance, not by the shape of the bearer. The Claude Messages +ingress attaches the stored native-main credential to a translated turn so forward sidecars stay +reachable, and that credential is indistinguishable from a caller's own by inspection. It is never +request-scoped: a translated Claude turn resolves through Pool selection like any other, so native +main keeps its health, quarantine and refresh-and-classify handling. Only a bearer the client +itself supplied is caller-owned and exempt from stored state. Both synchronous and asynchronous stored-main substitution in `src/codex/auth-context.ts` remove a caller account header before copying the stored identity; an absent stored account ID leaves no account header. Caller-owned native Direct authentication retains its existing passthrough behavior. `src/providers/openai-sidecar.ts` releases quota-probe ownership on every materialization or usability failure before transferring a resolved context to its caller. @@ -226,6 +232,21 @@ model settings, and noncanonical `openai` rows never receive that recovery path. terminal 403 codes as `needsReauth`; generic permission failures remain non-terminal, and a successful main usage refresh clears the runtime mark. +`src/codex/chatgpt-refresh-failure.ts` classifies token-endpoint failures for both stored-pool +and native-main refresh. Exact structured terminal codes apply only to authentication-error +statuses; HTTP429/5xx and malformed responses remain transient. Description-only OAuth400 +compatibility never overrides a structured code. Endpoint diagnostics contain fixed outcome, +HTTP status and an allowlisted code, never provider descriptions or credential material. Pool +refresh errors retain that same safe status/code metadata without changing cooldown classification. + +A native-main refusal is stored by physical auth path and refresh-grant fingerprint in a bounded +process-local set (64 oldest-first entries). Ordinary quarantine clears and successful usage polls +do not erase it; replacement grants are independently usable, and returning to a refused profile +retains its refusal until process restart or bounded eviction. Both selection and automatic +materialization refuse a matching grant. Late failures publish only after snapshot/cancellation +checks; downstream callers do not recreate account-wide quarantine. The account DTO checks and +projects refusal under native-profile ownership, and typed selection causes drive sign-in guidance. + Canonical forwarding alone can apply the optional client-output safety-buffering hint filter; API-key and custom forward destinations preserve their metadata. See [Responses transport](../transports/responses.md). diff --git a/tests/codex-integration/codex-account-store-refresh-classification.test.ts b/tests/codex-integration/codex-account-store-refresh-classification.test.ts index 9365a267686..61efdad3844 100644 --- a/tests/codex-integration/codex-account-store-refresh-classification.test.ts +++ b/tests/codex-integration/codex-account-store-refresh-classification.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test, beforeEach, afterEach } from "bun:test"; +import { describe, expect, test, beforeEach, afterEach, spyOn } from "bun:test"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -47,7 +47,8 @@ describe("codex refresh-failure classification", () => { }); /** Drive one forced refresh against a stubbed token endpoint and return the thrown reason. */ - async function classify(accountId: string, respond: () => Response): Promise { + async function classify(accountId: string, respond: () => Response, + metadata?: { status: number; code: string | undefined }): Promise { const { forceRefreshCodexPoolToken, readCodexAccountRecord, saveCodexAccountCredential, TokenRefreshError } = await import("../../src/codex/account-store"); saveCodexAccountCredential(accountId, { @@ -67,6 +68,7 @@ describe("codex refresh-failure classification", () => { throw new Error("expected a TokenRefreshError"); } catch (error) { expect(error).toBeInstanceOf(TokenRefreshError); + if (metadata) expect(error).toMatchObject(metadata); return (error as InstanceType).reason; } finally { globalThis.fetch = originalFetch; @@ -151,4 +153,47 @@ describe("codex refresh-failure classification", () => { const reason = await classify("unparseable", () => new Response("502 revoked", { status: 502 })); expect(reason).toBe("unknown"); }); + + test.each([429, 500, 503])("HTTP %s cannot retire a grant from terminal code or prose", async status => { + for (const [label, body] of [ + ["code", { error: "invalid_grant" }], + ["prose", { error_description: "refresh token was revoked and expired" }], + ] as const) { + expect(await classify(`availability-${status}-${label}`, () => Response.json(body, { status }))).toBe("unknown"); + } + }); + + test.each(["token_invalidated", "refresh_token_reused"])("exact %s retires a pool grant", async code => { + expect(await classify(code, () => Response.json({ error: { code } }, { status: 400 }))).toBe("revoked"); + }); + + test.each([123, {}, [], null])("malformed error_description %j stays transient", async error_description => { + expect(await classify(`malformed-${JSON.stringify(error_description)}`, () => Response.json({ error_description }, { status: 400 }))).toBe("unknown"); + }); + + test("pool refresh logs only status and recognized codes, never provider text", async () => { + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((...args) => { lines.push(args.join(" ")); }); + try { + await classify("diagnostic-known", () => Response.json({ error: "invalid_grant", error_description: "private-body-marker" }, { status: 400 })); + await classify("diagnostic-unknown", () => Response.json({ error: "private-code-marker", error_description: "private-body-marker" }, { status: 400 })); + expect(lines).toContain("[codex] pool refresh: reauth status=400 code=invalid_grant"); + expect(lines).toContain("[codex] pool refresh: transient status=400 code=none"); + expect(lines.join("\n")).not.toContain("private-body-marker"); + expect(lines.join("\n")).not.toContain("private-code-marker"); + expect(lines.join("\n")).not.toContain("grant-diagnostic"); + } finally { warn.mockRestore(); } + }); + + + test.each([ + { status: 400, bodyCode: "invalid_grant", code: "invalid_grant", reason: "revoked" }, + { status: 503, bodyCode: "server_error", code: "server_error", reason: "unknown" }, + { status: 400, bodyCode: "private-code-marker", code: undefined, reason: "unknown" }, + ])("pool refresh errors preserve safe structured metadata: %j", async ({ status, bodyCode, code, reason }) => { + expect(await classify(`metadata-${status}-${bodyCode}`, + () => Response.json({ error: bodyCode, error_description: "private-description-marker" }, { status }), + { status, code })).toBe(reason); + }); + }); diff --git a/tests/codex-integration/codex-account-unusable-reason.test.ts b/tests/codex-integration/codex-account-unusable-reason.test.ts index ca71c4adc9a..2486e3c4dea 100644 --- a/tests/codex-integration/codex-account-unusable-reason.test.ts +++ b/tests/codex-integration/codex-account-unusable-reason.test.ts @@ -1,3 +1,4 @@ +import { CodexPoolAuthenticationError } from "../../src/codex/auth-context"; import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -8,8 +9,9 @@ import { } from "../../src/codex/account-usability"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE } from "../../src/server/responses/codex-auth-error"; import { MAIN_CODEX_ACCOUNT_ID, MainAccountTokenRefreshError } from "../../src/codex/main-account"; -import { nativeMainRefreshFailureResponse } from "../../src/server/responses/codex-auth-error"; +import { mapCodexAuthContextErrorToResponse, nativeMainRefreshFailureResponse } from "../../src/server/responses/codex-auth-error"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -155,6 +157,21 @@ describe("native main refresh refusal", () => { const response = nativeMainRefreshFailureResponse(new MainAccountTokenRefreshError("reauth")); expect(response.status).toBe(401); const message = ((await response.json()) as { error: { message: string } }).error.message; - expect(message).toBe("Codex main account needs reauthentication"); + // Names the provider as the refuser and the command that fixes it: a bare "needs + // reauthentication" reads as a proxy fault for the operator whose plan was downgraded. + expect(message).toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + expect(message).toContain("codex login"); }); + + test("an unrelated pool refusal does not borrow the global main quarantine cause", async () => { + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + const ordinary = new CodexPoolAuthenticationError(); + const response = mapCodexAuthContextErrorToResponse(ordinary, { now: Date.now() })!; + expect(((await response.json()) as { error: { message: string } }).error.message).toBe(ordinary.message); + const main = mapCodexAuthContextErrorToResponse( + new CodexPoolAuthenticationError(undefined, { quarantinedMain: true }), { now: Date.now() }, + )!; + expect(((await main.json()) as { error: { message: string } }).error.message).toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + }); + }); diff --git a/tests/codex-integration/codex-main-grant-refusal.test.ts b/tests/codex-integration/codex-main-grant-refusal.test.ts new file mode 100644 index 00000000000..ebb1dca77ed --- /dev/null +++ b/tests/codex-integration/codex-main-grant-refusal.test.ts @@ -0,0 +1,319 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + clearAccountQuota, clearMainAccountInfoCache, listCodexAuthAccounts, +} from "../../src/codex/auth-api"; +import { + clearAccountNeedsReauth, isAccountNeedsReauth, markAccountNeedsReauth, +} from "../../src/codex/account-runtime-state"; +import { codexAccountUnusableReason } from "../../src/codex/account-usability"; +import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { codexCredentialMutationEpoch } from "../../src/codex/credential-mutation-epoch"; +import * as mainAccount from "../../src/codex/main-account"; +import { + forceRefreshMainAccountToken, getValidMainAccountToken, hasMainAccountRefreshGrant, + isMainAccountCredentialUsable, isMainAccountRefreshGrantRejected, MAIN_CODEX_ACCOUNT_ID, +} from "../../src/codex/main-account"; +import type { OcxConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +let root: string; +let home: string; +let previousCodexHome: string | undefined; +let previousOcxHome: string | undefined; +let warnings: string[]; +let warnSpy: ReturnType; +const cfg = { providers: {}, codexAccounts: [], activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID } as unknown as OcxConfig; + +function writeCredential(grant = "fixture-grant-a", target = home): string { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 86_400 })).toString("base64url"); + const access = `header.${payload}.signature`; + writeFileSync(join(target, "auth.json"), JSON.stringify({ + tokens: { access_token: access, refresh_token: grant, account_id: "fixture-main" }, + })); + return access; +} + +function mockEndpoint(reply: () => Response | Promise): string[] { + const grants: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.href !== "https://auth.openai.com/oauth/token") throw new Error("unexpected mocked endpoint"); + grants.push(new URLSearchParams(String(init?.body)).get("refresh_token") ?? ""); + return reply(); + }) as typeof fetch; + return grants; +} + +function terminalResponse(): Response { + return Response.json({ error: "invalid_grant", error_description: "private-description-marker" }, { status: 400 }); +} + +function expectRefused(): void { + expect(isMainAccountRefreshGrantRejected()).toBe(true); + expect(hasMainAccountRefreshGrant()).toBe(false); + expect(isMainAccountCredentialUsable()).toBe(false); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID)).toBe("needs_reauth"); +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "ocx-main-grant-refusal-")); + home = join(root, "codex-a"); + mkdirSync(home); + mkdirSync(join(root, "ocx")); + previousCodexHome = process.env.CODEX_HOME; + previousOcxHome = process.env.OPENCODEX_HOME; + process.env.CODEX_HOME = home; + process.env.OPENCODEX_HOME = join(root, "ocx"); + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + clearMainAccountInfoCache(); + clearAccountQuota(); + resetMainCodexAccountIdentityTrackingForTests(); + warnings = []; + warnSpy = spyOn(console, "warn").mockImplementation((...args: unknown[]) => warnings.push(args.map(String).join(" "))); + globalThis.fetch = (async () => { throw new Error("unexpected mocked endpoint"); }) as typeof fetch; +}); + +afterEach(() => { + warnSpy.mockRestore(); + globalThis.fetch = originalFetch; + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + clearMainAccountInfoCache(); + clearAccountQuota(); + resetMainCodexAccountIdentityTrackingForTests(); + mainAccount.setMainAccountPlan(null); + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOcxHome; + removeTreeWithRetry(root); +}); + +describe("native main refresh refusal is scoped to the physical path and grant", () => { + test("a refused fresh JWT and automatic forced retry make no further endpoint call", async () => { + const access = writeCredential(); + const before = readFileSync(join(home, "auth.json"), "utf8"); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expectRefused(); + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + await expect(getValidMainAccountToken()).rejects.toMatchObject({ reason: "reauth" }); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expect(grants).toEqual(["fixture-grant-a"]); + expect(readFileSync(join(home, "auth.json"), "utf8")).toBe(before); + expectRefused(); + }); + + test("successful explicit WHAM clears the generic mark without reviving the refused grant or DTO", async () => { + const access = writeCredential(); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + let whamCalls = 0; + const refreshFetch = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.pathname.endsWith("/wham/usage")) { + whamCalls += 1; + return Response.json({ plan_type: "plus", rate_limit: { primary_window: { used_percent: 10 } } }); + } + return refreshFetch(input, init); + }) as typeof fetch; + const accounts = await listCodexAuthAccounts(cfg, true); + expect(whamCalls).toBe(1); + expect(accounts.find(account => account.id === MAIN_CODEX_ACCOUNT_ID)) + .toMatchObject({ hasCredential: true, needsReauth: true }); + expectRefused(); + await expect(getValidMainAccountToken()).rejects.toMatchObject({ reason: "reauth" }); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expect(grants).toEqual(["fixture-grant-a"]); + }); + + test("a replacement written during a deferred refusal is not quarantined", async () => { + const access = writeCredential(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const grants = mockEndpoint(() => { entered.resolve(); return release.promise; }); + const epoch = codexCredentialMutationEpoch(); + const pending = forceRefreshMainAccountToken(access); + const outcome = pending.then(value => ({ value }), error => ({ error })); + try { + await entered.promise; + const replacement = writeCredential("fixture-grant-b"); + const replacedBytes = readFileSync(join(home, "auth.json"), "utf8"); + release.resolve(terminalResponse()); + expect(await outcome).toMatchObject({ error: { name: "MainAuthJsonChangedDuringRefreshError" } }); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(isMainAccountCredentialUsable()).toBe(true); + expect(codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID)).toBeUndefined(); + await expect(getValidMainAccountToken()).resolves.toMatchObject({ accessToken: replacement }); + expect(readFileSync(join(home, "auth.json"), "utf8")).toBe(replacedBytes); + expect(codexCredentialMutationEpoch()).toBe(epoch); + expect(grants).toEqual(["fixture-grant-a"]); + // Returning to the old grant also proves the stale refusal never published a record. + writeCredential("fixture-grant-a"); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + } finally { + release.resolve(terminalResponse()); + await pending.catch(() => {}); + } + }); + + test("a new grant after refusal is eligible and its successful refresh leaves the old refusal intact", async () => { + const access = writeCredential(); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + const replacement = writeCredential("fixture-grant-b"); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(hasMainAccountRefreshGrant()).toBe(true); + expect(isMainAccountCredentialUsable()).toBe(true); + await expect(getValidMainAccountToken()).resolves.toMatchObject({ accessToken: replacement }); + expect(grants).toEqual(["fixture-grant-a"]); + const replacementGrants = mockEndpoint(() => Response.json({ + access_token: "replacement-access", refresh_token: "fixture-grant-b", expires_in: 3600, + })); + await expect(forceRefreshMainAccountToken(replacement)).resolves.toMatchObject({ accessToken: "replacement-access" }); + expect(replacementGrants).toEqual(["fixture-grant-b"]); + writeCredential("fixture-grant-a"); + expectRefused(); + await expect(getValidMainAccountToken()).rejects.toMatchObject({ reason: "reauth" }); + expect(replacementGrants).toEqual(["fixture-grant-b"]); + }); + + test("profile A to B to A remembers A while the same grant bytes at B remain eligible", async () => { + const access = writeCredential(); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + const homeB = join(root, "codex-b"); + mkdirSync(homeB); + const accessB = writeCredential("fixture-grant-a", homeB); + process.env.CODEX_HOME = homeB; + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(isMainAccountCredentialUsable()).toBe(true); + expect(hasMainAccountRefreshGrant()).toBe(true); + await expect(getValidMainAccountToken()).resolves.toMatchObject({ accessToken: accessB }); + process.env.CODEX_HOME = home; + expectRefused(); + await expect(getValidMainAccountToken()).rejects.toMatchObject({ reason: "reauth" }); + expect(grants).toEqual(["fixture-grant-a"]); + }); + + test("a caller abort before a deferred terminal response publishes no refusal or credential mutation", async () => { + const access = writeCredential(); + const before = readFileSync(join(home, "auth.json"), "utf8"); + const epoch = codexCredentialMutationEpoch(); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const grants = mockEndpoint(() => { entered.resolve(); return release.promise; }); + const controller = new AbortController(); + const reason = new Error("fixture caller cancelled"); + const pending = forceRefreshMainAccountToken(access, { signal: controller.signal }); + const outcome = pending.then(value => ({ value }), error => ({ error })); + try { + await entered.promise; + controller.abort(reason); + release.resolve(terminalResponse()); + expect(await outcome).toEqual({ error: reason }); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(isMainAccountCredentialUsable()).toBe(true); + expect(readFileSync(join(home, "auth.json"), "utf8")).toBe(before); + expect(codexCredentialMutationEpoch()).toBe(epoch); + await expect(getValidMainAccountToken()).resolves.toMatchObject({ accessToken: access }); + expect(grants).toEqual(["fixture-grant-a"]); + } finally { + release.resolve(terminalResponse()); + await pending.catch(() => {}); + } + }); + + test.each([ + { nativeMainSelectionOnly: true }, + { requestOwnedMainCredential: true, isMainAccountTokenLive: () => true }, + ].flatMap(options => [false, true].map(ordinaryReauth => ({ options, ordinaryReauth }))))( + "selection fences do not inspect stored refusal: %j", async ({ options, ordinaryReauth }) => { + const access = writeCredential(); + mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + if (ordinaryReauth) markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + const refusalRead = spyOn(mainAccount, "isMainAccountRefreshGrantRejected"); + const grantRead = spyOn(mainAccount, "hasMainAccountRefreshGrant"); + const credentialRead = spyOn(mainAccount, "isMainAccountCredentialUsable"); + try { + expect(codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID, options)).toBeUndefined(); + expect(refusalRead).not.toHaveBeenCalled(); + expect(grantRead).not.toHaveBeenCalled(); + expect(credentialRead).not.toHaveBeenCalled(); + } finally { + refusalRead.mockRestore(); + grantRead.mockRestore(); + credentialRead.mockRestore(); + } + }); +}); + +describe("default native token endpoint classification and safe diagnostics", () => { + test.each([ + [400, { error: "invalid_grant" }, "invalid_grant"], + [401, { error: { code: "token_invalidated", message: "private-message-marker" } }, "token_invalidated"], + [400, { error: "refresh_token_expired", error_description: { marker: "private-description-marker" } }, "refresh_token_expired"], + [401, { error: "refresh_token_reused", error_description: 42 }, "refresh_token_reused"], + ] as const)("terminal %i exposes only its recognized code %j", async (status, body, code) => { + const access = writeCredential(); + mockEndpoint(() => Response.json(body, { status })); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expectRefused(); + expect(warnings).toContain(`[codex] native main refresh: reauth status=${status} code=${code}`); + expect(warnings.join("\n")).not.toMatch(/private-(?:message|description)-marker|fixture-grant-a|header\./); + }); + + test.each([ + [429, JSON.stringify({ error: "invalid_grant" })], + [503, JSON.stringify({ error: "refresh_token_expired", error_description: "private-description-marker" })], + [400, JSON.stringify({ error_description: 42 })], + [400, JSON.stringify({ error: "private code marker", error_description: "private-description-marker" })], + [400, JSON.stringify({ error: "private_token_shaped_marker", error_description: 42 })], + [400, JSON.stringify({ error_description: { marker: "private-description-marker" } })], + [400, "malformed-private-body-marker"], + ] as const)("%i with untrusted body %s remains transient", async (status, body) => { + const access = writeCredential(); + const grants = mockEndpoint(() => new Response(body, { status })); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "transient" }); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(hasMainAccountRefreshGrant()).toBe(true); + expect(isMainAccountCredentialUsable()).toBe(true); + expect(codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID)).toBeUndefined(); + expect(grants).toEqual(["fixture-grant-a"]); + expect(warnings.some(line => line.startsWith(`[codex] native main refresh: transient status=${status}`))).toBe(true); + expect(warnings.join("\n")).not.toMatch(/private|malformed|fixture-grant-a|header\./); + }); + + test.each(["revoked", "expired"])("valid description-only OAuth 400 retains %s compatibility without logging prose", async reason => { + const access = writeCredential(); + const grants = mockEndpoint(() => Response.json({ + error_description: `refresh token ${reason} private-description-marker`, + }, { status: 400 })); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expectRefused(); + expect(grants).toEqual(["fixture-grant-a"]); + expect(warnings).toContain("[codex] native main refresh: reauth status=400 code=none"); + expect(warnings.join("\n")).not.toMatch(/private-description-marker|refresh token|fixture-grant-a|header\./); + }); + + test("a transport error containing terminal prose remains transient without logging its message", async () => { + const access = writeCredential(); + const grants = mockEndpoint(() => { throw new Error("invalid_grant expired private-transport-marker"); }); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "transient" }); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(isMainAccountCredentialUsable()).toBe(true); + expect(grants).toEqual(["fixture-grant-a"]); + expect(warnings.join("\n")).not.toContain("private-transport-marker"); + }); +}); diff --git a/tests/codex-integration/main-account-hard-lock-recovery.test.ts b/tests/codex-integration/main-account-hard-lock-recovery.test.ts index 94cdefa694b..72f7bac7a25 100644 --- a/tests/codex-integration/main-account-hard-lock-recovery.test.ts +++ b/tests/codex-integration/main-account-hard-lock-recovery.test.ts @@ -12,7 +12,7 @@ import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTra import { clearAccountNeedsReauth, isAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; import { captureMainQuotaWriter, clearMainAccountInfoCache, getMainAccountInfoCache, setMainAccountInfoCache } from "../../src/codex/main-account-cache"; import { getMainAccountHardLockStatus } from "../../src/codex/main-account-hard-lock"; -import { setMainAccountPlan } from "../../src/codex/main-account"; +import { isMainAccountRefreshGrantRejected, setMainAccountPlan } from "../../src/codex/main-account"; import { clearAccountQuota, getAccountQuota, getMainPolicyQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; import { clearCodexUpstreamHealth, getCodexQuotaHealthSnapshot, recordCodexUpstreamOutcome } from "../../src/codex/routing"; import { flushConfigDirHardeningForTests } from "../../src/config/paths"; @@ -810,7 +810,8 @@ describe("main hard-lock background recovery", () => { await runMainAccountHardLockRecovery(config()); expect(calls).toEqual([tokenUrl]); expect(getMainPolicyQuota()).toEqual(retained); - expect(isAccountNeedsReauth(MAIN)).toBe(kind === "terminal"); + expect(isMainAccountRefreshGrantRejected()).toBe(kind === "terminal"); + expect(isAccountNeedsReauth(MAIN)).toBe(false); expect(getNativeMainProfileRequestCount()).toBe(0); if (kind === "terminal") { await runMainAccountHardLockRecovery(config()); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 4bc48733b57..b4ebe8527ab 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -10,6 +10,7 @@ "zed-retained-budget.test.ts": "providers", "jev-decision-scope.test.ts": "server", "codex-main-token-invalidated.test.ts": "codex-integration", + "codex-main-grant-refusal.test.ts": "codex-integration", "codex-shim-path-readiness.test.ts": "codex-integration", "codex-shim-fnm.test.ts": "codex-integration", "claude-intercept-cleanup.test.ts": "claude-integration", diff --git a/tests/responses/responses-native-main-refresh.test.ts b/tests/responses/responses-native-main-refresh.test.ts index 3a318cc0830..64afbb92e74 100644 --- a/tests/responses/responses-native-main-refresh.test.ts +++ b/tests/responses/responses-native-main-refresh.test.ts @@ -1,22 +1,40 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as fs from "node:fs"; import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { clearAccountNeedsReauth } from "../../src/codex/auth-api"; +import { clearAccountNeedsReauth, clearMainAccountInfoCache, listCodexAuthAccounts } from "../../src/codex/auth-api"; +import { CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE } from "../../src/server/responses/codex-auth-error"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; -import { isAccountNeedsReauth } from "../../src/codex/account-runtime-state"; -import { getValidMainAccountToken, MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; +import { isAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { codexAccountUnusableReason } from "../../src/codex/account-usability"; +import { getValidMainAccountToken, isMainAccountRefreshGrantRejected, MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; import { withNativeMainSharedClaim } from "../../src/codex/native-main-claim"; import type { NativeProfileContext } from "../../src/codex/native-profile-store"; import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; import { resolveResponsesApiAuth } from "../../src/server/auth-cors"; import { tryAdmitTurn } from "../../src/server/lifecycle"; import { handleResponses, handleResponsesCompact } from "../../src/server/responses"; +import { handleClaudeMessages } from "../../src/server/claude-messages"; +import { clearComboTargetCooldowns } from "../../src/combos/failover"; +import { clearComboSelectionState } from "../../src/combos/resolve"; import type { RequestLogContext } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; +import { captureCallerDirectAuth } from "../../src/providers/caller-authorization"; +import { fakeChatGptJwt } from "../helpers/agent-task-recovery"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +/** Smallest complete Anthropic stream: the combo's second target has to actually answer. */ +const ANTHROPIC_SSE = [ + ["message_start", { type: "message_start", message: { id: "msg_fallback", type: "message", role: "assistant", model: "m2", content: [], stop_reason: null, usage: { input_tokens: 1, output_tokens: 0 } } }], + ["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }], + ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "served" } }], + ["content_block_stop", { type: "content_block_stop", index: 0 }], + ["message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }], + ["message_stop", { type: "message_stop" }], +].map(([event, data]) => `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`).join(""); + const originalFetch = globalThis.fetch; let home = ""; let previousOcxHome: string | undefined; @@ -65,6 +83,9 @@ beforeEach(() => { clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); clearThreadAccountMap(); + clearComboTargetCooldowns(); + clearComboSelectionState(); + clearMainAccountInfoCache(); writeFileSync(join(home, "auth.json"), JSON.stringify({ tokens: { access_token: "rejected-access", @@ -83,6 +104,9 @@ afterEach(() => { clearAccountNeedsReauth(OTHER_ACCOUNT_ID); clearCodexUpstreamHealth(); clearThreadAccountMap(); + clearComboTargetCooldowns(); + clearComboSelectionState(); + clearMainAccountInfoCache(); if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOcxHome; if (previousCodexHome === undefined) delete process.env.CODEX_HOME; @@ -166,6 +190,61 @@ describe("native main 401 refresh and replay", () => { }, ); + test.each([false, true])("caller Direct 401 stays caller-owned when injected token bytes match=%s", async sameBytes => { + const caller = fakeChatGptJwt("caller-account"); + const stored = sameBytes ? caller : fakeChatGptJwt("stored-account"); + writeFileSync(join(home, "auth.json"), JSON.stringify({ + tokens: { access_token: stored, refresh_token: "stored-grant", account_id: sameBytes ? "caller-account" : "stored-account" }, + })); + const cfg = config(); + const callerDirectAuth = captureCallerDirectAuth(new Headers({ authorization: `Bearer ${caller}` }), cfg); + expect(callerDirectAuth).not.toBeNull(); + const sends: Headers[] = []; + const refreshes: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.hostname === "auth.openai.com") { + refreshes.push(url.pathname); + return Response.json({ error: "invalid_grant" }, { status: 400 }); + } + if (!url.pathname.endsWith("/responses")) throw new Error("unexpected mocked endpoint"); + sends.push(new Headers(init?.headers)); + return Response.json({ error: { code: "token_invalidated", message: "caller credential refused" } }, { status: 401 }); + }) as typeof fetch; + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + const originalRead = fs.readFileSync as (...args: unknown[]) => unknown; + let authReads = 0; + const readSpy = spyOn(fs, "readFileSync").mockImplementation(((...args: unknown[]) => { + if (typeof args[0] === "string" && args[0].endsWith("auth.json")) authReads += 1; + return originalRead(...args); + }) as typeof fs.readFileSync); + try { + // Calibrate the read observation before dispatch, then require zero native reads. + readFileSync(join(home, "auth.json")); + expect(authReads).toBe(1); + authReads = 0; + const response = await handleResponses(request("/v1/responses"), cfg, + { model: "", provider: "" } as RequestLogContext, { + turnAdmissionLease: turn!, callerDirectAuth, + stripClaudeMainAuthForNoncanonicalForward: true, + trustedClaudeMainAuth: { authorization: `Bearer ${stored}`, chatgptAccountId: sameBytes ? "caller-account" : "stored-account" }, + }); + expect(response.status).toBe(401); + await response.text(); + expect(sends).toHaveLength(1); + expect(sends[0]!.get("authorization")).toBe(`Bearer ${caller}`); + expect(sends[0]!.get("chatgpt-account-id")).toBe("caller-account"); + expect(authReads).toBe(0); + expect(refreshes).toEqual([]); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + } finally { + readSpy.mockRestore(); + turn?.release(); + } + expect(isMainAccountRefreshGrantRejected()).toBe(false); + }); + test("refreshes a refresh-only native main credential before upstream I/O", async () => { writeFileSync(join(home, "auth.json"), JSON.stringify({ tokens: { refresh_token: "refresh-grant", account_id: "account-main" }, @@ -315,6 +394,131 @@ describe("native main 401 refresh and replay", () => { }); } + /** + * The revoked-session harness: the access-token JWT still looks live, every send earns + * `token_invalidated`, and only the forced refresh can prove the credential is dead. The + * token endpoint's answer to that refresh is the variable. + */ + function installRevokedSessionHarness(refusal: { status: number; body: unknown }): { + sends: string[]; + refreshes: string[]; + } { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 86_400 })).toString("base64url"); + writeFileSync(join(home, "auth.json"), JSON.stringify({ + tokens: { + access_token: `header.${payload}.signature`, + refresh_token: "refresh-grant", + account_id: "account-main", + }, + })); + const sends: string[] = []; + const refreshes: string[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.hostname === "auth.openai.com") { + refreshes.push(new URLSearchParams(String(init?.body)).get("refresh_token") ?? ""); + return Response.json(refusal.body, { status: refusal.status }); + } + if (!url.pathname.endsWith("/responses")) { + return Response.json({ rate_limit: { primary_window: { used_percent: 10 } } }); + } + sends.push(new Headers(init?.headers).get("authorization") ?? ""); + return Response.json({ + error: { + message: "private-upstream-body-marker: token invalidated", + type: "invalid_request_error", + code: "token_invalidated", + }, + }, { status: 401 }); + }) as typeof fetch; + return { sends, refreshes }; + } + + // Production only ever confirmed bare 400 `invalid_grant`. These are the siblings upstream + // also emits for a revoked or downgraded session; each must retire the grant, not just the + // one code the incident happened to produce. + test.each([ + [400, "invalid_grant"], + [401, "refresh_token_invalidated"], + [401, "token_invalidated"], + [401, "refresh_token_expired"], + [401, "refresh_token_reused"], + ] as const)( + "retires native main on %i %s so the next request never sends", + async (status, code) => { + const { sends, refreshes } = installRevokedSessionHarness({ status, body: { error: code } }); + const refreshLines: string[] = []; + const warnSpy = spyOn(console, "warn").mockImplementation((...args: unknown[]) => { + refreshLines.push(args.map(String).join(" ")); + }); + + let first: Response; + try { + first = await handleResponses( + request("/v1/responses"), + config(), + { model: "", provider: "" } as RequestLogContext, + ); + } finally { + warnSpy.mockRestore(); + } + expect(first.status).toBe(401); + expect(JSON.parse(await first.text()).error.message).toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + // The verdict is on the record, with the endpoint's own status and code and nothing else. + // A revoked session is otherwise invisible: no write, and a status like any other 401. + expect(refreshLines).toContain(`[codex] native main refresh: reauth status=${status} code=${code}`); + expect(sends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(isMainAccountRefreshGrantRejected()).toBe(true); + expect(codexAccountUnusableReason(config(), MAIN_CODEX_ACCOUNT_ID)).toBe("needs_reauth"); + + const second = await handleResponses( + request("/v1/responses"), + config(), + { model: "", provider: "" } as RequestLogContext, + ); + // The point of the quarantine: the second request is refused locally, so a combo's next + // target is reached without paying another upstream round trip on a dead credential -- and + // it says the same actionable thing the discovering request said, not "no usable credential". + expect(second.status).toBe(401); + expect(JSON.parse(await second.text()).error.message).toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + expect(sends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + }, + ); + + test("keeps a 5xx whose description reads terminal transient and leaves the grant alive", async () => { + // The other half of the classification, and the reason it reads the structured code rather + // than the formatted message: upstream puts arbitrary prose in `error_description`, so a + // token-endpoint 5xx that happens to say "session expired" must stay a retryable refusal. + // Quarantining on it would retire a live grant and demand a sign-in nobody needs. + const { sends, refreshes } = installRevokedSessionHarness({ + status: 503, + body: { error: "server_error", error_description: "session expired or revoked; retry" }, + }); + + const transientLines: string[] = []; + const warnSpy = spyOn(console, "warn").mockImplementation((...args: unknown[]) => { + transientLines.push(args.map(String).join(" ")); + }); + let response: Response; + try { + response = await handleResponses( + request("/v1/responses"), + config(), + { model: "", provider: "" } as RequestLogContext, + ); + } finally { + warnSpy.mockRestore(); + } + expect(transientLines).toContain("[codex] native main refresh: transient status=503 code=server_error"); + expect(response.status).toBe(503); + expect(sends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + }); + test.each(["/v1/responses", "/v1/responses/compact"] as const)( "%s keeps the WebSocket string-abort claim cancellation as 499 without quarantining main", async path => { @@ -369,4 +573,178 @@ describe("native main 401 refresh and replay", () => { } }, ); + + /** + * The shape that actually runs in production: Claude Code POSTs /v1/messages with + * `model: combo/codexfirst`, whose first leg is the openai pool-main provider. The inbound + * wire is claude-messages, so that leg is TRANSLATED rather than passed through -- it + * dispatches through `prepareAdapterExchange`, not the passthrough route, and that dispatch + * recorded no Codex account outcome and attempted no refresh on a pre-stream 401. The + * revoked session was therefore rediscovered by a fresh upstream 401 on every request. + */ + test("a /v1/messages combo leg retires the dead grant instead of re-earning its 401", async () => { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 86_400 })).toString("base64url"); + writeFileSync(join(home, "auth.json"), JSON.stringify({ + tokens: { + access_token: `header.${payload}.signature`, + refresh_token: "refresh-grant", + account_id: "account-main", + }, + })); + const codexSends: string[] = []; + const refreshes: string[] = []; + let fallbackSends = 0; + let whamSends = 0; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.hostname === "auth.openai.com") { + refreshes.push(new URLSearchParams(String(init?.body)).get("refresh_token") ?? ""); + return Response.json({ error: "token_invalidated" }, { status: 401 }); + } + if (url.hostname === "fallback.test") { + fallbackSends += 1; + return new Response(ANTHROPIC_SSE, { headers: { "content-type": "text/event-stream" } }); + } + if (url.pathname.endsWith("/wham/usage")) { + whamSends += 1; + return Response.json({ plan_type: "plus", rate_limit: { primary_window: { used_percent: 10 } } }); + } + if (url.hostname !== "chatgpt.com" || !url.pathname.endsWith("/responses")) { + throw new Error("unexpected mocked endpoint"); + } + codexSends.push(new Headers(init?.headers).get("authorization") ?? ""); + return Response.json({ + error: { + message: "private-upstream-body-marker: token invalidated", + type: "invalid_request_error", + code: "token_invalidated", + }, + }, { status: 401 }); + }) as typeof fetch; + + const comboConfig = { + ...config(), + providers: { + ...config().providers, + fallback: { + adapter: "anthropic", + baseUrl: "https://fallback.test", + apiKey: "k", + }, + }, + combos: { + codexfirst: { + strategy: "failover", + targets: [ + { provider: "openai", model: "gpt-5.5" }, + { provider: "fallback", model: "m2" }, + ], + }, + }, + } as unknown as OcxConfig; + const messagesRequest = (): Request => new Request("http://localhost/v1/messages", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "combo/codexfirst", + max_tokens: 128, + messages: [{ role: "user", content: "hi" }], + }), + }); + + // The stored-main enrichment in handleClaudeMessages is gated on a claimed native-main + // profile for the turn, which needs a real admission lease. Without one the enrichment never + // fires and the leg resolves to `main-pool` -- which is exactly why every earlier attempt to + // reproduce the Mini from a leaseless harness came out already fixed. + // A fresh turn lease per request: reading a response to completion releases its lease, and a + // reused inactive lease would refuse the main-profile claim before refresh or send, letting the + // send-count assertions below pass without exercising the quarantine at all. + const turns: NonNullable>[] = []; + const freshTurn = () => { + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + turns.push(turn!); + return turn!; + }; + const warnings: string[] = []; + const warnSpy = spyOn(console, "warn").mockImplementation((...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }); + try { + let first: Response; + try { + first = await handleClaudeMessages( + messagesRequest(), + comboConfig, + { model: "", provider: "" } as RequestLogContext, + { requestId: `combo-401-${crypto.randomUUID()}`, start: Date.now(), turnAdmissionLease: freshTurn() }, + ); + await first.text(); + } finally { + warnSpy.mockRestore(); + } + // One failure is one line, and it names the refusal rather than reprinting the envelope. + // The raw body is multi-line JSON; a log built from it spread a single 401 over the screen. + const comboLine = warnings.find(line => line.startsWith("[combo] codexfirst: openai/")); + expect(comboLine).toBeDefined(); + expect(comboLine).not.toContain("\n"); + expect(comboLine).not.toContain("{"); + expect(comboLine).toContain("401"); + expect(warnings.join("\n")).not.toContain("private-upstream-body-marker"); + expect(warnings).toContain("[codex] native main refresh: reauth status=401 code=token_invalidated"); + // The combo still serves the turn from its second target; that was never the problem. + expect(first.status).toBe(200); + expect(fallbackSends).toBe(1); + // One send, one refresh attempt, and the refusal retired the grant for good. + expect(codexSends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(isMainAccountRefreshGrantRejected()).toBe(true); + expect(codexAccountUnusableReason(config(), MAIN_CODEX_ACCOUNT_ID)).toBe("needs_reauth"); + + // Re-offer the openai leg. Without this the combo's own target cooldown skips it and the + // assertions below would pass whether or not the account was ever quarantined. + clearComboTargetCooldowns(); + clearComboSelectionState(); + const second = await handleClaudeMessages( + messagesRequest(), + comboConfig, + { model: "", provider: "" } as RequestLogContext, + { requestId: `combo-401-${crypto.randomUUID()}`, start: Date.now(), turnAdmissionLease: freshTurn() }, + ); + await second.text(); + expect(second.status).toBe(200); + expect(fallbackSends).toBe(2); + // THE POINT: the quarantined main account is not sent to again, and its dead grant is not + // retried either. Before the fix this was a second upstream 401, three hundred times over. + expect(codexSends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + + // A real explicit WHAM success clears its generic mark, but cannot revive this grant. + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + clearMainAccountInfoCache(); + const whamBefore = whamSends; + const accounts = await listCodexAuthAccounts(comboConfig, true); + expect(whamSends).toBe(whamBefore + 1); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + expect(accounts.find(account => account.id === MAIN_CODEX_ACCOUNT_ID)?.needsReauth).toBe(true); + expect(isMainAccountRefreshGrantRejected()).toBe(true); + clearComboTargetCooldowns(); + clearComboSelectionState(); + const third = await handleClaudeMessages( + messagesRequest(), + comboConfig, + { model: "", provider: "" } as RequestLogContext, + { requestId: `combo-401-${crypto.randomUUID()}`, start: Date.now(), turnAdmissionLease: freshTurn() }, + ); + await third.text(); + expect(third.status).toBe(200); + expect(fallbackSends).toBe(3); + expect(codexSends).toHaveLength(1); + expect(refreshes).toEqual(["refresh-grant"]); + } finally { + warnSpy.mockRestore(); + for (const turn of turns) turn.release(); + } + }); }); diff --git a/tests/routing/router-blocked-cross-provider.test.ts b/tests/routing/router-blocked-cross-provider.test.ts index a51eb3530d2..2fd3175835d 100644 --- a/tests/routing/router-blocked-cross-provider.test.ts +++ b/tests/routing/router-blocked-cross-provider.test.ts @@ -77,12 +77,12 @@ describe("blocked-model redirect compatibility and provider changes", () => { }); const options = { admission: { kind: "environment", source: "dedicated" } } as HandleResponsesOptions; const redirected = routeModel(config({ m1: "google/g1" }), "openai/m1"); - const scoped = codexRouteCredentialDomainHeaders(req, redirected, options, false); + const scoped = codexRouteCredentialDomainHeaders(req, redirected, options, false, config({ m1: "google/g1" })); expect(scoped.get("authorization")).toBeNull(); expect(scoped.get("chatgpt-account-id")).toBeNull(); expect(scoped.get("x-extra")).toBe("kept"); const legacy = routeModel(config({ m1: "m2" }), "openai/m1"); - expect(codexRouteCredentialDomainHeaders(req, legacy, options, false).get("authorization")) + expect(codexRouteCredentialDomainHeaders(req, legacy, options, false, config({ m1: "m2" })).get("authorization")) .toBe("Bearer source-route-token"); }); From d3321369894de6fa8ae05d65c7424e6348e38efd Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 4 Oct 2026 01:51:13 +0900 Subject: [PATCH 2/5] fix(codex): retain caller ownership across preview selection --- src/codex/routing/selection.ts | 8 +++++--- src/server/responses/request-prepare.ts | 2 ++ structure/providers/openai-accounts.md | 5 ++++- tests/codex-integration/codex-main-grant-refusal.test.ts | 3 +++ tests/responses/responses-preview-main-read-fence.test.ts | 6 +++++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/codex/routing/selection.ts b/src/codex/routing/selection.ts index 2cd06226a0f..98fa2ac8c12 100644 --- a/src/codex/routing/selection.ts +++ b/src/codex/routing/selection.ts @@ -16,7 +16,7 @@ import { } from "../pool-rotation"; import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota, resetAtToMs } from "../quota"; import { codexPlanKey } from "../plan"; -import { MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan, hasMainAccountRefreshGrant } from "../main-account"; +import { MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "../main-account"; import type { OcxConfig } from "../../types"; import { CODEX_FAILURE_WINDOW_MS, computeCodexUsageScore } from "./cooldown-math"; import { @@ -220,7 +220,6 @@ export function getEligiblePoolAccounts( if ( excludeId !== MAIN_CODEX_ACCOUNT_ID && !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID) - && (!isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) || hasMainAccountRefreshGrant()) && getCodexQuotaHealthSnapshot(MAIN_CODEX_ACCOUNT_ID, quotaScope, now) === null && !isCodexAccountSoftAvoided(MAIN_CODEX_ACCOUNT_ID, now) // The main login is not in `config.codexAccounts`, so it never passes through the @@ -539,7 +538,7 @@ export function sharedStateSelectionOptions( selectionOptions?: CodexAccountUsabilityOptions, ): Pick< CodexAccountUsabilityOptions, - "nativeMainSelectionOnly" | "isMainAccountTokenLive" + "nativeMainSelectionOnly" | "isMainAccountTokenLive" | "requestOwnedMainCredential" > | undefined { if (!selectionOptions) return undefined; return { @@ -549,6 +548,9 @@ export function sharedStateSelectionOptions( ...(selectionOptions.isMainAccountTokenLive ? { isMainAccountTokenLive: selectionOptions.isMainAccountTokenLive } : {}), + ...(selectionOptions.requestOwnedMainCredential !== undefined + ? { requestOwnedMainCredential: selectionOptions.requestOwnedMainCredential } + : {}), }; } diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index 41755eacad8..2f251c26b74 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -737,6 +737,7 @@ export async function prepareResponsesRequest( // recovery or a profile drain fences the physical identity, final auth drops main and preview // has to drop it too. A hardcoded `true` would be wrong in the second case and `false` in // the first. + requestOwnedMainCredential: previewRequestScopedMainCredential, isMainAccountTokenLive: previewRequestScopedMainCredential ? () => previewRequestOwnedMainCredentialLive : undefined, @@ -995,6 +996,7 @@ export async function prepareResponsesRequest( const recoverySelectionOptions = { nativeMainSelectionOnly: !recoveryNativeMainBlocked && recoverySelectionAdmission?.mainProfileDraining === true, + requestOwnedMainCredential: recoveryRequestScopedMainCredential, isMainAccountTokenLive: recoveryRequestScopedMainCredential ? () => recoveryRequestOwnedMainCredentialLive : undefined, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 46f386eec27..37b40ee2342 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -201,7 +201,10 @@ sidecar candidate and cannot hide a failed Codex credential with separately bill `src/server/audio-upstream.ts` uses the same selection for standalone transcription. Explicit native Direct auth remains caller-owned; proxy-key-only Direct claims stored main before materialization, replacing both bearer and account identity exclusively from that credential. -Credential OWNERSHIP is decided by provenance, not by the shape of the bearer. The Claude Messages +Credential OWNERSHIP is decided by provenance, not by the shape of the bearer. Initial and +recovery previews and shared-state selection projections retain the caller-owned read fence; +`src/codex/routing/selection.ts` delegates main quarantine/grant eligibility to the canonical +`src/codex/account-usability.ts` checks instead of reading a refresh grant independently. The Claude Messages ingress attaches the stored native-main credential to a translated turn so forward sidecars stay reachable, and that credential is indistinguishable from a caller's own by inspection. It is never request-scoped: a translated Claude turn resolves through Pool selection like any other, so native diff --git a/tests/codex-integration/codex-main-grant-refusal.test.ts b/tests/codex-integration/codex-main-grant-refusal.test.ts index ebb1dca77ed..c46fbd6f643 100644 --- a/tests/codex-integration/codex-main-grant-refusal.test.ts +++ b/tests/codex-integration/codex-main-grant-refusal.test.ts @@ -1,3 +1,4 @@ +import { sharedStateSelectionOptions } from "../../src/codex/routing/selection"; import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -235,6 +236,8 @@ describe("native main refresh refusal is scoped to the physical path and grant", test.each([ { nativeMainSelectionOnly: true }, { requestOwnedMainCredential: true, isMainAccountTokenLive: () => true }, + sharedStateSelectionOptions({ requestOwnedMainCredential: true, isMainAccountTokenLive: () => true, + modelEligibleAccountIds: new Set(["some-pool-account"]) })!, ].flatMap(options => [false, true].map(ordinaryReauth => ({ options, ordinaryReauth }))))( "selection fences do not inspect stored refusal: %j", async ({ options, ordinaryReauth }) => { const access = writeCredential(); diff --git a/tests/responses/responses-preview-main-read-fence.test.ts b/tests/responses/responses-preview-main-read-fence.test.ts index f84b52ec615..bb3a95a7024 100644 --- a/tests/responses/responses-preview-main-read-fence.test.ts +++ b/tests/responses/responses-preview-main-read-fence.test.ts @@ -1,3 +1,4 @@ +import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import * as fs from "node:fs"; import { mkdtempSync, unlinkSync, writeFileSync } from "node:fs"; @@ -231,6 +232,7 @@ beforeEach(() => { process.env.OPENCODEX_HOME = testDir; process.env.CODEX_HOME = testDir; Date.now = () => NOW; + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -265,6 +267,7 @@ afterEach(() => { readSpy = undefined; if (blockedHomeId !== null) completeNativeMainRecovery(blockedHomeId); blockedHomeId = null; + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -341,8 +344,9 @@ describe("preview and final authentication agree on the native-main read fence", * main here -- so only a read count can distinguish a closed fence from a lucky outcome. The * stacks are asserted rather than the number so a failure names the caller that reopened it. */ - test("caller-owned preview reads no physical main credential through pool eligibility", async () => { + test.each([false, true])("caller-owned preview reads no physical main credential through pool eligibility (ordinary reauth=%s)", async ordinaryReauth => { seedMainDenial(); + if (ordinaryReauth) markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); calibrateMainReadCounter(); const upstreamAuth: Array = []; globalThis.fetch = (async (input, init) => { From 80b13f800b9d65e8c5ee8d459508d722daade193 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 4 Oct 2026 01:51:13 +0900 Subject: [PATCH 3/5] fix(codex): cancel alternate main refresh with its caller --- scripts/test-layout/layout.json | 1 + src/server/responses/core-codex-account.ts | 1 + tests/fixtures/test-layout-expected.json | 1 + ...ponses-alternate-main-cancellation.test.ts | 229 ++++++++++++++++++ 4 files changed, 232 insertions(+) create mode 100644 tests/responses/responses-alternate-main-cancellation.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index d4b0cd7ebbd..0f516e39cf1 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -15,6 +15,7 @@ "jev-decision-scope.test.ts": "server", "codex-main-token-invalidated.test.ts": "codex-integration", "codex-main-grant-refusal.test.ts": "codex-integration", + "responses-alternate-main-cancellation.test.ts": "responses", "codex-shim-path-readiness.test.ts": "codex-integration", "codex-shim-fnm.test.ts": "codex-integration", "claude-intercept-cleanup.test.ts": "claude-integration", diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index 6b0540938ef..2c57bb798da 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -641,6 +641,7 @@ export async function retryCodexPoolOnAlternateAccount( "pool", { excludeAccountId: firstAuthCtx.accountId, + signal: options.abortSignal, admission: options.admission, codexAuthPolicy: options.codexAuthPolicy, modelId: route.modelId, diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index b4ebe8527ab..d2a4998013d 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -11,6 +11,7 @@ "jev-decision-scope.test.ts": "server", "codex-main-token-invalidated.test.ts": "codex-integration", "codex-main-grant-refusal.test.ts": "codex-integration", + "responses-alternate-main-cancellation.test.ts": "responses", "codex-shim-path-readiness.test.ts": "codex-integration", "codex-shim-fnm.test.ts": "codex-integration", "claude-intercept-cleanup.test.ts": "claude-integration", diff --git a/tests/responses/responses-alternate-main-cancellation.test.ts b/tests/responses/responses-alternate-main-cancellation.test.ts new file mode 100644 index 00000000000..c59a6952787 --- /dev/null +++ b/tests/responses/responses-alternate-main-cancellation.test.ts @@ -0,0 +1,229 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearAccountNeedsReauth, clearAccountQuota, clearMainAccountInfoCache } from "../../src/codex/auth-api"; +import { isAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { resolveCodexAuthContext } from "../../src/codex/auth-context"; +import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { codexCredentialMutationEpoch } from "../../src/codex/credential-mutation-epoch"; +import { isMainAccountRefreshGrantRejected, MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "../../src/codex/main-account"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; +import { createTranslatorBudget, type TranslatorBudget } from "../../src/lib/translator-budget"; +import { parseRequest } from "../../src/responses/parser"; +import { handleClaudeMessages } from "../../src/server/claude-messages"; +import { codexAccountSelectionForTurn, tryAdmitTurn } from "../../src/server/lifecycle"; +import { retryCodexPoolOnAlternateAccount } from "../../src/server/responses/core-codex-account"; +import type { OcxConfig } from "../../src/types"; +import { fakeChatGptJwt } from "../helpers/agent-task-recovery"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +const POOL_ID = "alternate-cancel-pool"; +let root: string; +let authPath: string; +let previousCodexHome: string | undefined; +let previousOcxHome: string | undefined; +let releaseSpendHome: (() => void) | undefined; +let config: OcxConfig; + +function quotaRefusal(): Response { + return Response.json({ error: { code: "usage_limit_reached", message: "fixture pool quota exhausted" } }, { + status: 429, headers: { "retry-after": "60" }, + }); +} + +function refreshReply(terminal: boolean): Response { + return terminal + ? Response.json({ error: "invalid_grant" }, { status: 400 }) + : Response.json({ access_token: "fixture-refreshed-main", refresh_token: "fixture-rotated-main", expires_in: 3600 }); +} + +function completedInference(): Response { + const response = { + id: "resp_alternate_main", object: "response", status: "completed", model: "gpt-5.5", + output: [{ id: "msg_alternate_main", type: "message", role: "assistant", status: "completed", + content: [{ type: "output_text", text: "alternate answered", annotations: [] }] }], + usage: { input_tokens: 1, output_tokens: 2, total_tokens: 3 }, + }; + return new Response(`event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response })}\n\n`, { + headers: { "content-type": "text/event-stream" }, + }); +} + +function credentialState() { + return { + bytes: readFileSync(authPath, "utf8"), + epoch: codexCredentialMutationEpoch(), + refused: isMainAccountRefreshGrantRejected(), + genericReauth: isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID), + }; +} + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "ocx-alternate-main-cancel-")); + const codexHome = join(root, "codex"); + const ocxHome = join(root, "ocx"); + mkdirSync(codexHome); + mkdirSync(ocxHome); + previousCodexHome = process.env.CODEX_HOME; + previousOcxHome = process.env.OPENCODEX_HOME; + process.env.CODEX_HOME = codexHome; + process.env.OPENCODEX_HOME = ocxHome; + releaseSpendHome = acquireOwnedSpendHome(); + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + clearAccountNeedsReauth(POOL_ID); + clearAccountQuota(); + clearMainAccountInfoCache(); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + resetMainCodexAccountIdentityTrackingForTests(); + authPath = join(codexHome, "auth.json"); + writeFileSync(authPath, JSON.stringify({ tokens: { + access_token: fakeChatGptJwt("fixture-main-account", { exp: 1 }), + refresh_token: "fixture-main-grant", account_id: "fixture-main-account", + } })); + saveCodexAccountCredential(POOL_ID, { + accessToken: "fixture-pool-access", refreshToken: "fixture-pool-grant", + chatgptAccountId: "fixture-pool-account", expiresAt: Date.now() + 3_600_000, + }); + config = { + defaultProvider: "openai", activeCodexAccountId: POOL_ID, + autoSwitchThreshold: 0, accountPoolStrategy: "fill-first", + providers: { openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "pool", + baseUrl: "https://chatgpt.com/backend-api/codex" } }, + codexAccounts: [{ id: POOL_ID, label: "fixture pool" }], + } as OcxConfig; + globalThis.fetch = (async () => { throw new Error("unexpected mocked network call"); }) as typeof fetch; +}); + +afterEach(() => { + releaseSpendHome?.(); + releaseSpendHome = undefined; + globalThis.fetch = originalFetch; + clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + clearAccountNeedsReauth(POOL_ID); + clearAccountQuota(); + clearMainAccountInfoCache(); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + resetMainCodexAccountIdentityTrackingForTests(); + setMainAccountPlan(null); + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOcxHome; + removeTreeWithRetry(root); +}); + +type Turn = NonNullable>; +type Entry = "retry helper" | "messages ingress"; + +async function dispatch(entry: Entry, controller: AbortController, turn: Turn, budget: TranslatorBudget) { + if (entry === "messages ingress") { + // This is the actual ingress reconstruction: Claude creates a fresh Responses Request + // without its own signal, and must carry the original caller cancellation in options. + const req = new Request("http://localhost/v1/messages", { + method: "POST", headers: { "content-type": "application/json" }, signal: controller.signal, + body: JSON.stringify({ model: "openai/gpt-5.5", max_tokens: 64, stream: false, + messages: [{ role: "user", content: "hello" }] }), + }); + const response = await handleClaudeMessages(req, config, { model: "", provider: "" }, { + requestId: crypto.randomUUID(), start: Date.now(), turnAdmissionLease: turn, + }); + return { status: response.status, body: await response.text() }; + } + const callerAuthHeaders = new Headers(); + const firstAuthCtx = await resolveCodexAuthContext(callerAuthHeaders, config, "pool", { + modelId: "gpt-5.5", signal: controller.signal, + beginCodexAccountSelection: codexAccountSelectionForTurn(turn), + }); + expect(firstAuthCtx.kind).toBe("pool"); + if (firstAuthCtx.kind !== "pool") throw new Error("fixture did not select the first pool account"); + expect(firstAuthCtx.accountId).toBe(POOL_ID); + const result = await retryCodexPoolOnAlternateAccount({ + callerAuthHeaders, config, firstAuthCtx, firstResponse: quotaRefusal(), outcomeStatus: 429, + route: { providerName: "openai", modelId: "gpt-5.5", provider: config.providers.openai! }, + parsed: parseRequest({ model: "gpt-5.5", input: "hello", stream: false }), + logCtx: { model: "", provider: "" }, + options: { abortSignal: controller.signal, translatorBudget: budget, turnAdmissionLease: turn }, + upstream: controller, connectMs: 10_000, stream: false, httpOnly: true, + }); + if (result.kind !== "retried") return { kind: result.kind }; + expect(result.authCtx.kind).toBe("main-pool"); + return { kind: result.kind, status: result.upstreamResponse.status, body: await result.upstreamResponse.text() }; +} + +for (const entry of ["retry helper", "messages ingress"] as const) { + describe(`pool to native main cancellation through ${entry}`, () => { + for (const scenario of ["cancel-success", "cancel-terminal", "success-control"] as const) { + test(scenario, async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const refreshes: string[] = []; + const inference: string[] = []; + const controller = new AbortController(); + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + const budget = createTranslatorBudget(); + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.href === "https://auth.openai.com/oauth/token") { + refreshes.push(new URLSearchParams(String(init?.body)).get("refresh_token") ?? ""); + entered.resolve(); + // Deliberately ignore init.signal: a late endpoint result must be fenced by + // the credential owner even if the network operation does not stop on abort. + return release.promise; + } + if (url.hostname === "chatgpt.com" && url.pathname.endsWith("/responses")) { + const bearer = new Headers(init?.headers).get("authorization") ?? ""; + inference.push(bearer); + if (bearer === "Bearer fixture-pool-access") return quotaRefusal(); + if (bearer === "Bearer fixture-refreshed-main") return completedInference(); + throw new Error("unexpected inference credential"); + } + throw new Error(`unexpected mocked endpoint: ${url.origin}${url.pathname}`); + }) as typeof fetch; + const before = credentialState(); + expect(before.refused).toBe(false); + expect(before.genericReauth).toBe(false); + const outcome = dispatch(entry, controller, turn!, budget).then( + value => ({ value }), error => ({ error }), + ); + try { + // A premature return reports a fixture failure instead of hanging on a gate. + const reached = await Promise.race([ + entered.promise.then(() => ({ phase: "refresh" })), + outcome.then(result => ({ phase: "finished-before-refresh", result })), + ]); + expect(reached).toEqual({ phase: "refresh" }); + expect(refreshes).toEqual(["fixture-main-grant"]); + const initialInference = entry === "messages ingress" ? ["Bearer fixture-pool-access"] : []; + expect(inference).toEqual(initialInference); + if (scenario !== "success-control") controller.abort(new Error("fixture caller cancelled")); + release.resolve(refreshReply(scenario === "cancel-terminal")); + const result = await outcome; + if (scenario === "success-control") { + expect(result).toMatchObject({ value: { status: 200 } }); + expect(inference).toEqual([...initialInference, "Bearer fixture-refreshed-main"]); + const after = credentialState(); + expect(after.bytes).toContain("fixture-rotated-main"); + expect(after.epoch).toBe(before.epoch + 1); + expect(after.refused).toBe(false); + } else { + // One aggregate assertion reports every forbidden side effect, including + // terminal refusal that leaves bytes intact but poisons later requests. + expect({ ...credentialState(), inference }).toEqual({ ...before, inference: initialInference }); + } + } finally { + release.resolve(refreshReply(false)); + await outcome; + turn?.release(); + budget.dispose(); + } + }); + } + }); +} From 06034e44f2802d002e7ff525847782d2133fb161 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 4 Oct 2026 03:29:26 +0900 Subject: [PATCH 4/5] fix(codex): keep main refusal guidance and refresh diagnostics coherent --- src/codex/account-usability.ts | 13 +- src/codex/auth-context.ts | 18 ++- src/codex/main-account.ts | 38 +++++- structure/providers/openai-accounts.md | 5 +- .../codex-main-grant-refusal.test.ts | 126 ++++++++++++++++++ 5 files changed, 180 insertions(+), 20 deletions(-) diff --git a/src/codex/account-usability.ts b/src/codex/account-usability.ts index 9ca84db8b14..df7ef92d5be 100644 --- a/src/codex/account-usability.ts +++ b/src/codex/account-usability.ts @@ -2,9 +2,7 @@ import { readCodexAccountRecord } from "./account-store"; import { isAccountNeedsReauth } from "./account-runtime-state"; import { MAIN_CODEX_ACCOUNT_ID, - hasMainAccountRefreshGrant, - isMainAccountCredentialUsable, - isMainAccountRefreshGrantRejected, + getMainAccountCredentialStatus, isMainAccountTokenLive, } from "./main-account"; import { hasLegacyMainCodexPoolAccount, isSelectableCodexPoolAccount } from "./account-id"; @@ -15,7 +13,7 @@ import { isMainAccountHardLocked } from "./main-account-hard-lock"; export interface CodexAccountUsabilityOptions { /** Route using cached runtime state only; the caller must reject selected main before auth. */ nativeMainSelectionOnly?: boolean; - /** Test seam for proving whether routing attempted a physical native-token read. */ + /** Liveness override; ownership/read fences independently control physical credential reads. */ isMainAccountTokenLive?: typeof isMainAccountTokenLive; /** Confirmed account ids for an account-gated model; omitted for ordinary native models. */ modelEligibleAccountIds?: ReadonlySet; @@ -95,12 +93,13 @@ export function codexAccountUnusableReason( if (options.requestOwnedMainCredential) { return options.isMainAccountTokenLive?.() === false ? "main_credential_unavailable" : undefined; } - if (isMainAccountRefreshGrantRejected()) return "needs_reauth"; - if (isAccountNeedsReauth(accountId) && !hasMainAccountRefreshGrant()) return "needs_reauth"; + const credential = getMainAccountCredentialStatus(); + if (credential.refreshGrantRejected) return "needs_reauth"; + if (isAccountNeedsReauth(accountId) && !credential.hasRefreshGrant) return "needs_reauth"; // Main account: a refresh grant is enough to route; materialization refreshes before I/O. const mainLive = options.isMainAccountTokenLive ? options.isMainAccountTokenLive() - : isMainAccountCredentialUsable(); + : credential.usable; return mainLive ? undefined : "main_credential_unavailable"; } const exists = (config.codexAccounts ?? []) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 99ef6aeec09..13ef41a0adb 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1318,7 +1318,12 @@ export async function resolveCodexAuthContext( "Selected Codex account does not support this model", ); } - throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); + const mainReason = fixedAccountId === MAIN_CODEX_ACCOUNT_ID && !nativeMainReadsForbidden + && !policy.pausedCodexAccountIds?.includes(fixedAccountId) + ? codexAccountUnusableReason(config, fixedAccountId, selectionOptions) : undefined; + throw new CodexPoolAuthenticationError("Selected Codex account is unavailable", { + quarantinedMain: mainReason === "needs_reauth", + }); } // Recovery or a turn drain deliberately makes physical main unobservable. // If no healthy pool route is available, report the temporary fence rather @@ -1379,10 +1384,15 @@ export async function resolveCodexAuthContext( throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); } if (isAccountNeedsReauth(accountId)) { - throw new CodexPoolAuthenticationError("Selected Codex account needs reauthentication"); + throw new CodexPoolAuthenticationError("Selected Codex account needs reauthentication", { + quarantinedMain: accountId === MAIN_CODEX_ACCOUNT_ID && !nativeMainReadsForbidden, + }); } - if (!isCodexAccountUsable(config, accountId, selectionOptions)) { - throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); + const unusableReason = codexAccountUnusableReason(config, accountId, selectionOptions); + if (unusableReason !== undefined) { + throw new CodexPoolAuthenticationError("Selected Codex account is unavailable", { + quarantinedMain: accountId === MAIN_CODEX_ACCOUNT_ID && unusableReason === "needs_reauth", + }); } } } catch (cause) { diff --git a/src/codex/main-account.ts b/src/codex/main-account.ts index d62facb3245..107b486522d 100644 --- a/src/codex/main-account.ts +++ b/src/codex/main-account.ts @@ -121,8 +121,17 @@ async function refreshNativeMainGrant( noteChatgptRefreshFailure("native main", resp.status, failure); throw new MainAccountTokenRefreshError(failure.reason === "unknown" ? "transient" : "reauth"); } + let credentials: OAuthCredentials; + try { + credentials = credsFromToken((await resp.json()) as Record); + } catch { + if (options.signal.aborted) throw options.signal.reason; + noteNativeMainRefresh("transient", resp.status); + // Parsing errors may include provider body text; retain only the fixed verdict. + throw new MainAccountTokenRefreshError("transient"); + } noteNativeMainRefresh("ok", resp.status); - return credsFromToken((await resp.json()) as Record); + return credentials; } /** @@ -219,20 +228,35 @@ function credentialRefreshGrantRejected(current: MainAuthJsonCredential | null): return !!key && isMainRefreshGrantRejected(key); } +export interface MainAccountCredentialStatus { + refreshGrantRejected: boolean; + hasRefreshGrant: boolean; + usable: boolean; +} + +/** One owned physical observation; no credential material escapes this view. */ +export function getMainAccountCredentialStatus(now = Date.now()): MainAccountCredentialStatus { + const current = readMainAuthJsonCredential(); + const refreshGrantRejected = credentialRefreshGrantRejected(current); + const hasRefreshGrant = !!current?.refreshToken && !refreshGrantRejected; + return { + refreshGrantRejected, + hasRefreshGrant, + usable: !refreshGrantRejected && (hasRefreshGrant || mainAccessTokenFresh(current?.accessToken, now, 0)), + }; +} + /** Current owned profile only; callers must hold the native read/selection permission. */ export function isMainAccountRefreshGrantRejected(): boolean { - return credentialRefreshGrantRejected(readMainAuthJsonCredential()); + return getMainAccountCredentialStatus().refreshGrantRejected; } export function isMainAccountCredentialUsable(now = Date.now()): boolean { - const current = readMainAuthJsonCredential(); - return !credentialRefreshGrantRejected(current) - && (!!current?.refreshToken || mainAccessTokenFresh(current?.accessToken, now, 0)); + return getMainAccountCredentialStatus(now).usable; } export function hasMainAccountRefreshGrant(): boolean { - const current = readMainAuthJsonCredential(); - return !!current?.refreshToken && !credentialRefreshGrantRejected(current); + return getMainAccountCredentialStatus().hasRefreshGrant; } function assertMainAuthJsonSnapshotUnchanged(expected: MainAuthJsonCredential): void { diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 37b40ee2342..4ed82b37bca 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -237,7 +237,7 @@ successful main usage refresh clears the runtime mark. `src/codex/chatgpt-refresh-failure.ts` classifies token-endpoint failures for both stored-pool and native-main refresh. Exact structured terminal codes apply only to authentication-error -statuses; HTTP429/5xx and malformed responses remain transient. Description-only OAuth400 +statuses; HTTP 429/5xx and malformed responses remain transient. Description-only OAuth 400 compatibility never overrides a structured code. Endpoint diagnostics contain fixed outcome, HTTP status and an allowlisted code, never provider descriptions or credential material. Pool refresh errors retain that same safe status/code metadata without changing cooldown classification. @@ -246,7 +246,8 @@ A native-main refusal is stored by physical auth path and refresh-grant fingerpr process-local set (64 oldest-first entries). Ordinary quarantine clears and successful usage polls do not erase it; replacement grants are independently usable, and returning to a refused profile retains its refusal until process restart or bounded eviction. Both selection and automatic -materialization refuse a matching grant. Late failures publish only after snapshot/cancellation +materialization refuse a matching grant. Each stored-main usability check derives rejection, +grant availability and liveness from one physical snapshot, after the request read fences. Late failures publish only after snapshot/cancellation checks; downstream callers do not recreate account-wide quarantine. The account DTO checks and projects refusal under native-profile ownership, and typed selection causes drive sign-in guidance. diff --git a/tests/codex-integration/codex-main-grant-refusal.test.ts b/tests/codex-integration/codex-main-grant-refusal.test.ts index c46fbd6f643..1b3b6079edf 100644 --- a/tests/codex-integration/codex-main-grant-refusal.test.ts +++ b/tests/codex-integration/codex-main-grant-refusal.test.ts @@ -1,5 +1,6 @@ import { sharedStateSelectionOptions } from "../../src/codex/routing/selection"; import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as fs from "node:fs"; import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -17,6 +18,10 @@ import { forceRefreshMainAccountToken, getValidMainAccountToken, hasMainAccountRefreshGrant, isMainAccountCredentialUsable, isMainAccountRefreshGrantRejected, MAIN_CODEX_ACCOUNT_ID, } from "../../src/codex/main-account"; +import { headersForCodexAuthContext, resolveCodexAuthContext } from "../../src/codex/auth-context"; +import { codexAccountSelectionForTurn, tryAdmitTurn } from "../../src/server/lifecycle"; +import { CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE, mapCodexAuthContextErrorToResponse } from "../../src/server/responses/codex-auth-error"; +import { fakeChatGptJwt } from "../helpers/agent-task-recovery"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; @@ -247,15 +252,18 @@ describe("native main refresh refusal is scoped to the physical path and grant", const refusalRead = spyOn(mainAccount, "isMainAccountRefreshGrantRejected"); const grantRead = spyOn(mainAccount, "hasMainAccountRefreshGrant"); const credentialRead = spyOn(mainAccount, "isMainAccountCredentialUsable"); + const snapshotRead = spyOn(mainAccount, "getMainAccountCredentialStatus"); try { expect(codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID, options)).toBeUndefined(); expect(refusalRead).not.toHaveBeenCalled(); expect(grantRead).not.toHaveBeenCalled(); expect(credentialRead).not.toHaveBeenCalled(); + expect(snapshotRead).not.toHaveBeenCalled(); } finally { refusalRead.mockRestore(); grantRead.mockRestore(); credentialRead.mockRestore(); + snapshotRead.mockRestore(); } }); }); @@ -320,3 +328,121 @@ describe("default native token endpoint classification and safe diagnostics", () expect(warnings.join("\n")).not.toContain("private-transport-marker"); }); }); + + +describe("explicit account selection after a native endpoint refusal", () => { + test.each(["main", "missing pool", "paused main"] as const)("fixed %s maps only its own refusal", async selected => { + const access = writeCredential(); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + expectRefused(); + const accountId = selected === "missing pool" ? "fixture-missing-pool" : MAIN_CODEX_ACCOUNT_ID; + const config = { + ...cfg, + codexAccounts: selected === "missing pool" ? [{ id: accountId, label: "fixture missing" }] : [], + ...(selected === "paused main" ? { pausedCodexAccountIds: [MAIN_CODEX_ACCOUNT_ID] } : {}), + } as OcxConfig; + const turn = tryAdmitTurn(); + expect(turn).not.toBeNull(); + try { + const outcome = await resolveCodexAuthContext(new Headers(), config, "pool", { + accountId, modelId: "gpt-5.5", beginCodexAccountSelection: codexAccountSelectionForTurn(turn!), + }).then(value => ({ value }), error => ({ error })); + expect(outcome).toHaveProperty("error"); + if (!("error" in outcome)) throw new Error("fixed unavailable account unexpectedly resolved"); + const response = mapCodexAuthContextErrorToResponse(outcome.error, { accountSelector: selected, now: Date.now() }); + expect(response).toBeDefined(); + expect(response!.status).toBe(401); + const body = await response!.json() as { error: { message: string } }; + if (selected === "main") expect(body.error.message).toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + else expect(body.error.message).not.toBe(CODEX_MAIN_SIGN_IN_REQUIRED_MESSAGE); + expect(grants).toEqual(["fixture-grant-a"]); + expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + } finally { + turn?.release(); + } + }); + + test("caller-owned explicit main retains its own credential despite stored refusal", async () => { + const access = writeCredential(); + const grants = mockEndpoint(terminalResponse); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "reauth" }); + const caller = fakeChatGptJwt("fixture-caller-account"); + const incoming = new Headers({ authorization: `Bearer ${caller}`, "chatgpt-account-id": "fixture-caller-account" }); + const context = await resolveCodexAuthContext(incoming, cfg, "pool", { + accountId: MAIN_CODEX_ACCOUNT_ID, modelId: "gpt-5.5", requestScopedMainCredential: true, + }); + expect(context).toMatchObject({ kind: "main", accountId: null }); + const forwarded = headersForCodexAuthContext(incoming, context); + expect(forwarded.get("authorization")).toBe(`Bearer ${caller}`); + expect(forwarded.get("chatgpt-account-id")).toBe("fixture-caller-account"); + expect(grants).toEqual(["fixture-grant-a"]); + expectRefused(); + }); +}); + +describe("native refresh success diagnostics require a valid credential", () => { + const canary = "private-success-body-canary"; + test.each([ + ["malformed JSON", `{${canary}`], + ["missing access token", JSON.stringify({ marker: canary })], + ["numeric access token", JSON.stringify({ access_token: 42, marker: canary })], + ["object access token", JSON.stringify({ access_token: { marker: canary } })], + ["empty access token", JSON.stringify({ access_token: "", marker: canary })], + ["null body", "null"], + ] as const)("HTTP 200 with %s is transient without a success verdict or mutation", async (_label, body) => { + const access = writeCredential(); + const before = readFileSync(join(home, "auth.json"), "utf8"); + const epoch = codexCredentialMutationEpoch(); + const grants = mockEndpoint(() => new Response(body, { status: 200, headers: { "content-type": "application/json" } })); + await expect(forceRefreshMainAccountToken(access)).rejects.toMatchObject({ reason: "transient" }); + expect(grants).toEqual(["fixture-grant-a"]); + expect({ + bytes: readFileSync(join(home, "auth.json"), "utf8"), epoch: codexCredentialMutationEpoch(), + refused: isMainAccountRefreshGrantRejected(), genericReauth: isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID), + okLogs: warnings.filter(line => line.startsWith("[codex] native main refresh: ok")), + leakedBody: warnings.some(line => line.includes(canary)), + }).toEqual({ bytes: before, epoch, refused: false, genericReauth: false, okLogs: [], leakedBody: false }); + }); + + test("a validated HTTP 200 credential publishes once and records one safe success verdict", async () => { + const access = writeCredential(); + const epoch = codexCredentialMutationEpoch(); + const grants = mockEndpoint(() => Response.json({ + access_token: "fixture-validated-access", refresh_token: "fixture-validated-refresh", expires_in: 3600, + ignored_extra: canary, + })); + await expect(forceRefreshMainAccountToken(access)).resolves.toMatchObject({ accessToken: "fixture-validated-access" }); + expect(grants).toEqual(["fixture-grant-a"]); + expect(codexCredentialMutationEpoch()).toBe(epoch + 1); + expect(JSON.parse(readFileSync(join(home, "auth.json"), "utf8")).tokens) + .toMatchObject({ access_token: "fixture-validated-access", refresh_token: "fixture-validated-refresh" }); + expect(isMainAccountRefreshGrantRejected()).toBe(false); + expect(warnings.filter(line => line.startsWith("[codex] native main refresh: ok"))) + .toEqual(["[codex] native main refresh: ok status=200 code=none"]); + expect(warnings.join("\n")).not.toMatch(/private-success-body-canary|fixture-validated-access|fixture-validated-refresh/); + }); +}); + +test("main usability consumes one coherent physical snapshot under an ordinary reauth mark", () => { + const access = writeCredential(); + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + const originalRead = fs.readFileSync as (...args: unknown[]) => unknown; + let authReads = 0; + const readSpy = spyOn(fs, "readFileSync").mockImplementation(((...args: unknown[]) => { + if (typeof args[0] === "string" && args[0].endsWith("auth.json")) authReads += 1; + return originalRead(...args); + }) as typeof fs.readFileSync); + try { + expect(isMainAccountCredentialUsable()).toBe(true); + expect(authReads).toBe(1); + authReads = 0; + const reason = codexAccountUnusableReason(cfg, MAIN_CODEX_ACCOUNT_ID); + expect(reason).toBeUndefined(); + expect(warnings.join("\n")).not.toContain(access); + expect(warnings.join("\n")).not.toContain("fixture-grant-a"); + expect(authReads).toBe(1); + } finally { + readSpy.mockRestore(); + } +}); From d5eccb7c58843aba5d903f2f1aed1ae68fcf3814 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 4 Oct 2026 03:40:44 +0900 Subject: [PATCH 5/5] test(codex): guard coherent status reads under main hard lock --- tests/codex-integration/main-account-hard-lock-auth.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/codex-integration/main-account-hard-lock-auth.test.ts b/tests/codex-integration/main-account-hard-lock-auth.test.ts index c8cfc4090ad..a7fcfabe763 100644 --- a/tests/codex-integration/main-account-hard-lock-auth.test.ts +++ b/tests/codex-integration/main-account-hard-lock-auth.test.ts @@ -104,6 +104,7 @@ function forbidPhysicalReads(): void { spyOn(mainAccount, "getMainAccountToken").mockImplementation(forbidden); spyOn(mainAccount, "getValidMainAccountToken").mockImplementation(forbidden); spyOn(mainAccount, "isMainAccountCredentialUsable").mockImplementation(forbidden); + spyOn(mainAccount, "getMainAccountCredentialStatus").mockImplementation(forbidden); } function addAlternative(cfg: OcxConfig): void {