diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index 888a131f6fb..7d756a212d7 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -1047,6 +1047,8 @@ In **Codex Set → Multi-auth**, enable the **Codex credits** switch in the **Co When an account reaches 100% on a usage window and still holds credits, upstream keeps serving it and draws the balance. OpenCodex does not let that happen by default: an account at 100% is switched out while its weekly or monthly window (only monthly on 30-day plans) or its 5-hour window is full, and used again once that window resets. The order of the other accounts does not change, and when no other account is available, selection finds none rather than spending credits. A request for the main account is refused like a hard-lock refusal until the reset. A full 5-hour window without a reset time holds an account only while that reading is fresh. Allowing the main account does not lift its hard lock (on by default at 98%), which still stops it first; turn the lock off if the main account should spend credits (a lock at 100% still stops it at 100%). To let accounts keep working from their credits, turn on **Use credits** next to the **Codex credits** switch in the Codex Auth header. The switch allows every account. To choose accounts one by one, open an account card's **⋯** menu and use **Use credits after limit** there; the header switch shows a middle position when only some accounts are on. An account allowed to spend carries a **Uses credits** badge on its card. New accounts start off. The choice is stored in `creditCodexAccountIds` and never redeems reset credits; the **Codex credits** display switch only shows balances and never changes routing. +For an opted-in main account with a full usage window, the background recovery cycle renews previously spendable credit observations from three minutes of age, even while the dashboard is closed. It prepares a valid token before the authenticated usage lookup so the balance can be observed again before its five-minute freshness limit. Missing, zero, restricted or retracted credits do not trigger this renewal. A failed or incomplete lookup does not refresh old balance evidence, and requests are still refused once that evidence expires. The separate main-account hard lock and upstream retry delays still apply. This metadata lookup does not send a model validation request or set or clear the account's needs-reauth state. + Background revalidation is separate and off by default. It requires Token Guardian, the `openai` provider's `proactive` refresh policy, and `tokenGuardian.codexWarmupEnabled`. It skips accounts awaiting deferred registration validation. ### Cancelling main-account device reauthentication diff --git a/docs-site/src/content/docs/zh-cn/guides/codex-integration.md b/docs-site/src/content/docs/zh-cn/guides/codex-integration.md index 9b19850626b..925a226790f 100644 --- a/docs-site/src/content/docs/zh-cn/guides/codex-integration.md +++ b/docs-site/src/content/docs/zh-cn/guides/codex-integration.md @@ -327,6 +327,8 @@ fallback 行为，参见 [Sub-agent Surface](/guides/sub-agent-surface/)。 在 **Codex Set → Multi-auth** 中，打开 **Codex Auth** 标题栏中的 **Codex 额度** 开关，即可在 Week 下方显示主账号和各池账号最近查询到的积分。默认关闭，并保存为 `showCodexCredits`。余额按地区格式显示；上游报告时会显示无限额或超额使用上限警告。由于没有积分总上限，条形表示可用状态而非百分比。此开关仅控制显示，新登录需等待其自身的查询结果。 +允许主账号在额度耗尽后使用积分且用量窗口已满时，后台恢复周期会从观察满三分钟起更新此前可用的积分余额，即使仪表盘已关闭。查询前先准备有效令牌，为五分钟有效期到期前重新观察余额留出时间。缺失、零余额、受限或已撤回的积分不会触发更新。查询失败或信息不完整不会刷新旧余额证据；证据过期后请求仍会被拒绝。独立的主账号硬锁和上游重试等待期限仍然适用。这是已认证的用量查询，不发送模型验证请求，也不会设置或清除账号的 needs-reauth 状态。 + 后台重新验证是独立功能，默认关闭。它要求 Token Guardian、`openai` 的 `proactive` 刷新策略及 `tokenGuardian.codexWarmupEnabled`，并跳过等待注册验证的账号。 ### 账号停止处理请求的原因 diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 5a6f6c0f21f..edb2f1d7b72 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1158,7 +1158,7 @@ "main-account-hard-lock-auth.test.ts": "codex-integration", "main-account-hard-lock-default.test.ts": "codex-integration", "main-account-hard-lock-policy.test.ts": "codex-integration", - "main-account-hard-lock-recovery.test.ts": "codex-integration", + "main-account-hard-lock-recovery.test.ts": "codex-integration", "main-account-credit-renewal.test.ts": "codex-integration", "main-account-policy-binding-wait.test.ts": "codex-integration", "main-device-reauth-api.test.ts": "codex-integration", "main-device-reauth-ui.test.ts": "gui", diff --git a/src/codex/auth-api/main-account-probe.ts b/src/codex/auth-api/main-account-probe.ts index 80472a77929..5e208e5e1ec 100644 --- a/src/codex/auth-api/main-account-probe.ts +++ b/src/codex/auth-api/main-account-probe.ts @@ -146,6 +146,11 @@ export async function fetchMainAccountInfo(forceRefresh = false, config?: OcxCon return (await fetchMainAccountInfoSnapshot(forceRefresh, config)).info; } +export interface MainAccountInfoProbeOptions { + /** Metadata-only renewal never sets or clears the traffic reauth quarantine. */ + passive?: boolean; +} + export const EMPTY_MAIN_ACCOUNT_INFO: MainAccountInfo = { email: null, plan: null, quota: null }; export async function retryMainAccountInfoIfIdentityChanged( @@ -154,12 +159,13 @@ export async function retryMainAccountInfoIfIdentityChanged( nativeMainLease: AdmissionLease, explicitRefresh: boolean, paced: boolean, + options: MainAccountInfoProbeOptions = {}, ): Promise { const currentAccountId = getMainChatgptAccountId(); if (currentAccountId === null || currentAccountId === requestAccountId) return null; reconcileMainCodexAccountRuntimeState(); return retriesRemaining > 0 - ? fetchMainAccountInfoWhileOwned(true, retriesRemaining - 1, nativeMainLease, explicitRefresh, false, paced) + ? fetchMainAccountInfoWhileOwned(true, retriesRemaining - 1, nativeMainLease, explicitRefresh, false, paced, options) : { info: EMPTY_MAIN_ACCOUNT_INFO, credentialChecked: true, hasCredential: true }; } @@ -171,6 +177,7 @@ export async function fetchMainAccountInfoAttempt( explicitRefresh: boolean = forceRefresh, postReset = false, config?: OcxConfig, + options: MainAccountInfoProbeOptions = {}, ): Promise { const nativeMainLease = existingNativeMainLease ?? tryAcquireNativeMainProfileClaim(); if (!nativeMainLease) { @@ -184,7 +191,7 @@ export async function fetchMainAccountInfoAttempt( try { const paced = sharedMainQuotaPacingEnabled(config ?? loadConfig()); const operation = async () => ({ - ...await fetchMainAccountInfoWhileOwned(forceRefresh, retriesRemaining, nativeMainLease, explicitRefresh, postReset, paced), + ...await fetchMainAccountInfoWhileOwned(forceRefresh, retriesRemaining, nativeMainLease, explicitRefresh, postReset, paced, options), identityGeneration: captureMainAccountIdentityGeneration(), }); if (nativeMainSharedClaimHeld) return await operation(); @@ -225,6 +232,7 @@ export async function fetchMainAccountInfoWhileOwned( /** Reset-credit consume needs a fresh post-spend observation, not an earlier read. */ postReset = false, paced = true, + options: MainAccountInfoProbeOptions = {}, ): Promise { const writerGeneration = captureConfigGeneration(); reconcileMainCodexAccountRuntimeState(); @@ -287,7 +295,16 @@ export async function fetchMainAccountInfoWhileOwned( const current = credentialIsCurrent() && writerGeneration === captureConfigGeneration(); if (!current) return { info: getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO, credentialChecked: true, hasCredential: true }; - if (explicitRefresh && (admission.result.quotaRefresh?.status === "ok" + // Single-flight shares evidence, but each caller retains its own auth behavior. + if (!options.passive && admission.result.terminalAuthFailure) { + const diagnosticStillLive = admission.result.quotaRefreshGeneration !== undefined + && isMainAccountIdentityGenerationLive(admission.result.quotaRefreshGeneration); + clearMainAccountInfoCache(); + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID, writerGeneration); + return { ...admission.result, ...(diagnosticStillLive + ? { quotaRefreshGeneration: captureMainAccountIdentityGeneration() } : {}) }; + } + if (!options.passive && explicitRefresh && (admission.result.quotaRefresh?.status === "ok" || admission.result.quotaRefresh?.status === "not_reported")) clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); return admission.result; } @@ -297,13 +314,13 @@ export async function fetchMainAccountInfoWhileOwned( if (!resp.ok) { const authFailure = await classifyMainAuthResponse(resp, isMainAccountTokenVerifiablyLive()); const terminalAuthFailure = authFailure.terminal; - const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced); + const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced, options); if (retried) return retried; if (!isQuotaDispatchCurrent(dispatchSequence) || !credentialIsCurrent()) { return { info: getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO, credentialChecked: true, hasCredential: true }; } - if (terminalAuthFailure) { + if (terminalAuthFailure && !options.passive) { // Account for this attempt's own synchronous invalidation, never prior external drift. const diagnosticStillLive = isMainAccountIdentityGenerationLive(quotaRefreshGeneration); clearMainAccountInfoCache(); @@ -324,7 +341,7 @@ export async function fetchMainAccountInfoWhileOwned( quotaPhase = "body"; const data = (await resp.json()) as WhamUsageResponse; quotaPhase = "publish"; - const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced); + const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced, options); if (retried) return retried; quotaPhase = "decode"; if (data === null || typeof data !== "object" || Array.isArray(data)) { @@ -376,7 +393,7 @@ export async function fetchMainAccountInfoWhileOwned( // never settled and the dashboard kept showing nothing — the symptom #327 reported. // An explicit refresh is an operator asking to re-evaluate, normally right after // signing in again, so it stays authoritative. - if (explicitRefresh) clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + if (explicitRefresh && !options.passive) clearAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); // Mirror main quota + plan into the shared stores so the rotation engine can // score and auto-switch the main account exactly like a pool account (Option A). setMainAccountPlan(result.plan); @@ -398,7 +415,7 @@ export async function fetchMainAccountInfoWhileOwned( ...(freshResetCredits !== undefined ? { freshResetCredits } : {}), }; } catch (error) { - const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced); + const retried = await retryMainAccountInfoIfIdentityChanged(requestAccountId, retriesRemaining, nativeMainLease, explicitRefresh, paced, options); if (retried) return retried; if (!credentialIsCurrent()) { return { info: getMainAccountInfoCache() ?? EMPTY_MAIN_ACCOUNT_INFO, diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index e3c9393bff7..be8d93fdd93 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -1,9 +1,11 @@ -import { codexAccountUsesCreditsAfterLimit } from "../account-credit-use"; +import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "../account-credit-use"; import { nextCodexUsageQueryAt, nextQuotaQueryDelay, pruneRemovedCodexPoolUsageAccounts } from "../quota-query-backoff"; import { CODEX_PRIORITY_FAILBACK_REFRESH_MS } from "../account-priority"; import { codexQuotaHasFreshUsage } from "../quota-observation-freshness"; import { getCodexAccountCredential, getValidCodexToken, readCodexAccountRecord } from "../account-store"; -import { getAccountQuota, isCompleteCodexQuotaRecoverySnapshot } from "../quota"; +import { getAccountQuota, getMainPolicyQuota, isCompleteCodexQuotaRecoverySnapshot } from "../quota"; +import { isCodexAccountPaused } from "../account-pause"; +import { hasSpendableCodexCredits } from "../quota-types"; import { reconcileMainCodexAccountRuntimeState } from "../account-lifecycle"; import { claimDueCodexQuotaRecoveryProbes, settleCodexQuotaRecoveryProbe } from "../routing"; import { readCodexTokens } from "../auth-collision"; @@ -73,17 +75,33 @@ export async function runCodexCooldownRecoveryProbes(config: OcxConfig, now = Da return cooldownRecoveryInFlight; } +const MAIN_CREDIT_RENEWAL_AGE_MS = 3 * 60_000; + let mainHardLockRecoveryInFlight: Promise | null = null; let mainHardLockRecoveryAttempt: { identity: number; credential: number; after: number; delay: number } | undefined; /** Metadata-only recovery on the existing sweep; failures retain the observed policy block. */ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise { if (mainHardLockRecoveryInFlight) return mainHardLockRecoveryInFlight; + /** Schedule a real credit observation while leaving actual-time admission unchanged. */ + const creditRecoveryNeeded = (): boolean => { + const now = Date.now(); + const quota = getMainPolicyQuota(); + const observedAt = quota?.credits?.observedAt; + // getMainPolicyQuota binds evidence to the observed physical identity. Only renew + // valid formerly spendable funds; admission retains the actual five-minute clock. + return codexAccountUsesCreditsAfterLimit(config, MAIN_CODEX_ACCOUNT_ID) + && !isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID) + && codexUsageLimitResetAt(quota, undefined, now) !== undefined + && observedAt !== undefined && observedAt > 0 + && now - observedAt >= MAIN_CREDIT_RENEWAL_AGE_MS + && hasSpendableCodexCredits(quota, observedAt); + }; const status = getMainAccountHardLockStatus(config); - if (status.state !== "blocked") { mainHardLockRecoveryAttempt = undefined; return; } + if (status.state !== "blocked" && !creditRecoveryNeeded()) { mainHardLockRecoveryAttempt = undefined; return; } if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. - if (status.resetAt !== undefined && status.resetAt > Date.now()) return; + if (status.state === "blocked" && status.resetAt !== undefined && status.resetAt > Date.now()) return; // Reconcile the owned physical credential before skipping work for an older one. reconcileMainCodexAccountRuntimeState(); const physical = readCodexTokens(); @@ -98,9 +116,10 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise if (!lease) return; mainHardLockRecoveryInFlight = (async () => { reconcileMainCodexAccountRuntimeState(); - if (getMainAccountHardLockStatus(config).state !== "blocked" + if ((getMainAccountHardLockStatus(config).state !== "blocked" && !creditRecoveryNeeded()) || isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; const identityGeneration = captureMainAccountIdentityGeneration(); + const passive = getMainAccountHardLockStatus(config).state !== "blocked"; try { // Refresh can require an exclusive credential claim: never hold WHAM's shared // claim while obtaining a valid token. The runtime lease spans both operations. @@ -111,7 +130,12 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise // Native refresh retains its own grant-scoped refusal; global quarantine would outlive it. return; } - if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; + const currentStatus = getMainAccountHardLockStatus(config); + if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) + || !isMainAccountIdentityGenerationLive(identityGeneration) + || (passive && !creditRecoveryNeeded()) + || (currentStatus.state === "blocked" && currentStatus.resetAt !== undefined + && currentStatus.resetAt > Date.now())) return; const credential = getMainQuotaCredentialGeneration(); // Reconcile the physical bearer before consulting the previous credential's delay. if (previous?.identity === identityGeneration && previous.credential === credential @@ -123,7 +147,7 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise after: currentQueryAfter }; return; } - const result = await fetchMainAccountInfoAttempt(true, 1, lease, false, false, false, config); + const result = await fetchMainAccountInfoAttempt(true, 1, lease, false, false, false, config, { passive }); // Never charge a replacement credential for a late result from its predecessor. if (isMainAccountIdentityGenerationLive(identityGeneration) && credential === getMainQuotaCredentialGeneration()) { @@ -134,11 +158,11 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const authStatus = result.quotaRefresh.status === "http_error" ? result.quotaRefresh.httpStatus : undefined; const transientAuth = (authStatus === 401 || authStatus === 403) - && !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + && !result.terminalAuthFailure && !isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); if (transientAuth) mainHardLockRecoveryAttempt = undefined; else { const delay = nextQuotaQueryDelay(previousDelay || undefined); - mainHardLockRecoveryAttempt = getMainAccountHardLockStatus(config).state === "blocked" + mainHardLockRecoveryAttempt = getMainAccountHardLockStatus(config).state === "blocked" || creditRecoveryNeeded() ? { identity: identityGeneration, credential, delay, after: Math.max(Date.now() + delay, queryAfter) } : undefined; } diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index bb3e6df8ef6..7c44d37e8b3 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -479,6 +479,8 @@ false. Exact-account and Direct routes are unchanged. ## Main-account policy observations +Opted-in main-account credit renewal follows the [spendable credit contract](openai-tiers.md#spendable-codex-credits). The existing background sweep prepares a token before WHAM, rechecks eligibility, and uses a passive probe that neither sets nor clears needs-reauth, including after identity retries. Other probes keep their existing auth behavior. A non-passive caller joining renewal applies terminal-auth quarantine, or clears it on a successful explicit refresh, only while its credential and configuration fences remain current; shared evidence does not inherit the owner's passive intent. + The main-account admission policy defaults to 90% for short windows and 98% for long windows; `codexMainAccountHardLockThresholds` permits ordered integer thresholds from 80 through 100. Policy evidence retention uses the minimum configurable 80%, so a partial update or elapsed reset diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 99202c57eee..889bfd3c656 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -316,6 +316,22 @@ payloads cannot clear cooldowns, actual request refusals still drive cooldown/fa default-on main-account hard lock retains its separate local admission policy. Registration warmup remains conservative and does not spend paid credits to validate an exhausted account. +For an opted-in, unpaused main account with a currently full usage window, the existing +`src/codex/auth-api/pool-mode-gate.ts` recovery sweep renews credit observations from three minutes +of age, leaving time for token preparation before the five-minute freshness limit. Only valid, +previously spendable positive or unlimited evidence bound to the same physical account schedules +renewal; missing, zero, restricted or retracted credits do not. Admission still uses the actual +clock and original spending controls. Native profile ownership, generation fences, single-flight, +query pacing and failure backoff remain in force, including upstream Retry-After. Eligibility is +checked again after token preparation. The independent hard lock still applies. + +Credit renewal supplies the passive option to `src/codex/auth-api/main-account-probe.ts`, including +identity retries: WHAM success and terminal 401/403 responses never set or clear needs-reauth in +this mode. Native token preparation also preserves the traffic quarantine. Other callers retain +the existing auth behavior. Failed or incomplete observations never renew the credit clock; +refreshing usage does not redeem reset credits or validate pending accounts through inference. +Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. + Credit parsing, expiry, partial updates and reset-ticket separation are covered in `tests/codex-integration/codex-quota-parser-parity.test.ts`; selection and bulk-pause behavior are covered in `tests/codex-integration/codex-credits-after-limit.test.ts` and diff --git a/tests/codex-integration/main-account-credit-renewal.test.ts b/tests/codex-integration/main-account-credit-renewal.test.ts new file mode 100644 index 00000000000..f5a1504007a --- /dev/null +++ b/tests/codex-integration/main-account-credit-renewal.test.ts @@ -0,0 +1,400 @@ +import { afterEach, beforeEach, expect, mock, spyOn, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { CodexMainAccountCreditsOffError, resolveCodexAuthContext } from "../../src/codex/auth-context"; +import { fetchMainAccountInfoAttempt } from "../../src/codex/auth-api/main-account-probe"; +import { registerCodexCooldownRecoveryProbeWorker, runMainAccountHardLockRecovery } from "../../src/codex/auth-api/pool-mode-gate"; +import { MAIN_CODEX_ACCOUNT_ID as MAIN } from "../../src/codex/account-id"; +import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { clearAccountNeedsReauth, isAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { captureMainQuotaWriter, clearMainAccountInfoCache, getMainAccountInfoCache, isMainAccountIdentityGenerationLive, observeMainQuotaCredential, setMainAccountInfoCache } from "../../src/codex/main-account-cache"; +import { clearAccountQuota, getMainPolicyQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { hasSpendableCodexCredits, type CodexSpendableCredits } from "../../src/codex/quota-types"; +import { resetQuotaQueryBackoffForTests } from "../../src/codex/quota-query-backoff"; +import { clearCodexUpstreamHealth } from "../../src/codex/routing"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import * as mainAccount from "../../src/codex/main-account"; +import * as sweeper from "../../src/lib/state-store-sweeper"; +import * as quotaQueries from "../../src/codex/quota-query-backoff"; +import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; +import { getNativeMainProfileRequestCount, resetLifecycleDrainStateForTests } from "../../src/server/lifecycle"; +import { mapCodexAuthContextErrorToResponse } from "../../src/server/responses/codex-auth-error"; +import type { OcxConfig } from "../../src/types"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const accountId = "fixture-credit-renewal"; +const whamUrl = "https://chatgpt.com/backend-api/wham/usage"; +const tokenUrl = "https://auth.openai.com/oauth/token"; +const HOUR = 3_600_000; +let home: string; +let now: number; +let previousHome: string | undefined; +let previousCodexHome: string | undefined; +let previousFetch: typeof fetch; +let cfg: OcxConfig; +let calls: string[]; + +function bearer(id = accountId, expired = false): string { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(now / 1000) + (expired ? -120 : 86_400), + "https://api.openai.com/auth": { chatgpt_account_id: id } })).toString("base64url"); + return `header.${payload}.signature`; +} + +function writeMain(id = accountId, expired = false): void { + writeFileSync(join(home, "auth.json"), JSON.stringify({ tokens: { + access_token: bearer(id, expired), refresh_token: "fixture-refresh", account_id: id, + } })); + reconcileMainCodexAccountRuntimeState(); + observeMainQuotaCredential(bearer(id, expired), id); +} + +function seed(credits: CodexSpendableCredits | null | undefined = { + hasCredits: true, balance: 5, allowed: true, observedAt: now - 180_000, +}): void { + clearAccountQuota(MAIN); + setAccountQuotaFromParsed(MAIN, { weeklyPercent: 100, weeklyResetAt: now + HOUR, + ...(credits !== undefined ? { credits } : {}) }, undefined, captureMainQuotaWriter(accountId)); +} + +function usage(restriction?: string): Response { + return Response.json({ plan_type: "plus", rate_limit: { + primary_window: { used_percent: 100, limit_window_seconds: 604_800, + reset_at: Math.floor((now + HOUR) / 1000) }, + secondary_window: null, tertiary_window: null, + }, ...(restriction === "omitted" ? {} : { credits: restriction === "retracted" ? null : { + has_credits: restriction !== "empty", balance: restriction === "empty" ? 0 : 5, + overage_limit_reached: restriction === "overage", + } }), spend_control: { reached: restriction === "restricted" } }); +} + +function fetchWith(handler: (url: string, init?: RequestInit) => Promise): void { + globalThis.fetch = Object.assign(async (input: Parameters[0], init?: RequestInit) => { + const url = String(input); + calls.push(url); + expect([whamUrl, tokenUrl]).toContain(url); + expect(getNativeMainProfileRequestCount()).toBeGreaterThan(0); + return handler(url, init); + }, { preconnect: previousFetch.preconnect }); +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +async function refused(): Promise { + const error = await resolveCodexAuthContext(new Headers(), cfg, "pool").catch(error => error); + expect(error).toBeInstanceOf(CodexMainAccountCreditsOffError); + const response = mapCodexAuthContextErrorToResponse(error, { now })!; + expect(response.status).toBe(429); + expect(isAccountNeedsReauth(MAIN)).toBe(false); +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + previousFetch = globalThis.fetch; + home = mkdtempSync(join(tmpdir(), "ocx-credit-renewal-")); + process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = home; + now = Math.floor(Date.now() / 1000) * 1000; + spyOn(Date, "now").mockImplementation(() => now); + const aclOk = { success: true, exitCode: 0, timedOut: false, stdout: "" }; + setIcaclsRunnerForTests(() => aclOk); + setAsyncIcaclsRunnerForTests(async () => aclOk); + clearAccountQuota(); + clearAccountNeedsReauth(MAIN); + clearCodexUpstreamHealth(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); + resetLifecycleDrainStateForTests(); + resetQuotaQueryBackoffForTests(); + mainAccount.setMainAccountPlan(null); + writeMain(); + calls = []; + cfg = { port: 10100, defaultProvider: "openai", codexMainAccountHardLock: false, + activeCodexAccountId: MAIN, creditCodexAccountIds: [MAIN], codexAccounts: [], providers: { openai: { + adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", codexAccountMode: "pool", + } } }; + seed(); + fetchWith(async () => usage()); +}); + +afterEach(async () => { + mock.restore(); + globalThis.fetch = previousFetch; + clearAccountQuota(); + clearAccountNeedsReauth(MAIN); + clearCodexUpstreamHealth(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); + resetLifecycleDrainStateForTests(); + resetQuotaQueryBackoffForTests(); + mainAccount.setMainAccountPlan(null); + await flushConfigDirHardeningForTests(); + setIcaclsRunnerForTests(null); + setAsyncIcaclsRunnerForTests(null); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); +}); + +test("without renewal, five-minute credit expiry produces a local 429", async () => { + seed({ hasCredits: true, balance: 5, observedAt: now }); + await expect(resolveCodexAuthContext(new Headers(), cfg, "pool")).resolves.toMatchObject({ kind: "main-pool" }); + now += 300_001; + await refused(); + expect(calls).toEqual([]); +}); + +test.each([false, true])("hidden-dashboard sweep renews at three minutes and admits past original expiry (unlimited=%j)", async unlimited => { + const original = now; + seed({ hasCredits: true, ...(unlimited ? { unlimited: true } : { balance: 5 }), observedAt: now }); + let worker: (() => void) | undefined; + spyOn(sweeper, "registerStateSweepAfterTick").mockImplementation(registration => { worker = registration.afterTick; }); + registerCodexCooldownRecoveryProbeWorker(cfg); + now += 179_999; + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); + now++; + expect(worker).toBeDefined(); + worker!(); // No account-list/dashboard request is made. + await runMainAccountHardLockRecovery(cfg); // Join the already scheduled sweep. + expect(calls).toEqual([whamUrl]); + expect(getMainPolicyQuota()?.credits?.observedAt).toBe(now); + now = original + 300_001; + await expect(resolveCodexAuthContext(new Headers(), cfg, "pool")).resolves.toMatchObject({ kind: "main-pool" }); + expect(getNativeMainProfileRequestCount()).toBe(0); +}); + +test.each([503, 401, 403])("failed renewal HTTP %s preserves evidence and eventual refusal", async status => { + seed({ hasCredits: true, balance: 5, observedAt: now - 240_000 }); + const observation = getMainPolicyQuota()!.credits!.observedAt; + fetchWith(async () => Response.json({ detail: { code: "invalid_workspace_selected" } }, { status })); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl]); + expect(getMainPolicyQuota()?.credits?.observedAt).toBe(observation); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + now += 120_001; + await refused(); +}); + +test.each(["empty", "restricted", "overage", "retracted", "omitted"])("renewal with %s credits cannot authorize expired evidence", async restriction => { + fetchWith(async () => usage(restriction)); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl]); + now += 120_001; + await refused(); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl]); +}); + +test.each(["missing", "retracted", "zero", "negative", "invalid-balance", "no-flags", "no-balance", "has-false", "restricted", "overage", "future", "invalid-time", "negative-time"])("%s credit evidence never initiates renewal", async condition => { + const credits: CodexSpendableCredits = { hasCredits: true, balance: 5, observedAt: now - 180_000 }; + if (condition === "zero") credits.balance = 0; + if (condition === "negative") credits.balance = -1; + if (condition === "invalid-balance") credits.balance = Number.NaN; + if (condition === "no-flags") delete credits.hasCredits; + if (condition === "no-balance") delete credits.balance; + if (condition === "has-false") { credits.hasCredits = false; credits.unlimited = true; } + if (condition === "restricted") credits.allowed = false; + if (condition === "overage") credits.overageLimitReached = true; + if (condition === "future") credits.observedAt = now + 1; + if (condition === "invalid-time") credits.observedAt = Number.NaN; + if (condition === "negative-time") credits.observedAt = -1; + seed(condition === "retracted" ? null : credits); + if (condition === "missing") { clearAccountQuota(MAIN); setAccountQuotaFromParsed(MAIN, + { weeklyPercent: 100, weeklyResetAt: now + HOUR }, undefined, captureMainQuotaWriter(accountId)); } + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); + expect(getNativeMainProfileRequestCount()).toBe(0); +}); + +test.each(["no-consent", "paused", "reauth", "fresh", "reset", "hard-lock", "identity"])("%s excludes credit renewal", async condition => { + if (condition === "no-consent") cfg.creditCodexAccountIds = []; + if (condition === "paused") cfg.pausedCodexAccountIds = [MAIN]; + if (condition === "reauth") markAccountNeedsReauth(MAIN); + if (condition === "fresh") seed({ hasCredits: true, balance: 5, observedAt: now }); + if (condition === "reset") setAccountQuotaFromParsed(MAIN, { weeklyPercent: 0 }, undefined, captureMainQuotaWriter(accountId)); + if (condition === "hard-lock") cfg.codexMainAccountHardLock = true; + if (condition === "identity") writeMain("fixture-replacement"); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); +}); + +test("expired access token is refreshed before credit WHAM", async () => { + writeMain(accountId, true); + seed(); + fetchWith(async (url, init) => { + if (url === tokenUrl) return Response.json({ access_token: bearer(), refresh_token: "fixture-rotated", expires_in: 86_400 }); + expect(new Headers(init?.headers).get("authorization")).toBe(`Bearer ${bearer()}`); + return usage(); + }); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([tokenUrl, whamUrl]); + expect(getMainPolicyQuota()?.credits?.observedAt).toBe(now); + now += 120_001; + await expect(resolveCodexAuthContext(new Headers(), cfg, "pool")).resolves.toMatchObject({ kind: "main-pool" }); +}); + +test("failed refresh leaves stale-credit refusal and never calls WHAM", async () => { + writeMain(accountId, true); + seed(); + fetchWith(async () => Response.json({ error: "invalid_grant" }, { status: 400 })); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([tokenUrl]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + now += 120_001; + await refused(); +}); + +test.each(["consent", "pause", "credits", "reauth", "identity", "hard-lock"])("eligibility is checked after token preparation changes %s", async change => { + const original = mainAccount.getValidMainAccountToken; + spyOn(mainAccount, "getValidMainAccountToken").mockImplementation(async options => { + const token = await original(options); + if (change === "consent") cfg.creditCodexAccountIds = []; + if (change === "pause") cfg.pausedCodexAccountIds = [MAIN]; + if (change === "credits") seed(null); + if (change === "reauth") markAccountNeedsReauth(MAIN); + if (change === "identity") writeMain("fixture-replacement"); + if (change === "hard-lock") cfg.codexMainAccountHardLock = true; + return token; + }); + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); +}); + +test("overlapping sweeps share one credit renewal and release ownership", async () => { + const entered = deferred(); + const finish = deferred(); + fetchWith(async () => { entered.resolve(); return finish.promise; }); + const first = runMainAccountHardLockRecovery(cfg); + await entered.promise; + const others = [runMainAccountHardLockRecovery(cfg), runMainAccountHardLockRecovery(cfg)]; + expect(calls).toEqual([whamUrl]); + finish.resolve(usage()); + await Promise.all([first, ...others]); + expect(getNativeMainProfileRequestCount()).toBe(0); + expect(getMainPolicyQuota()?.credits?.observedAt).toBe(now); +}); + +test("upstream Retry-After paces credit renewal until its deadline", async () => { + let reads = 0; + fetchWith(async () => ++reads === 1 ? new Response("{}", { status: 429, headers: { "Retry-After": "900" } }) : usage()); + await runMainAccountHardLockRecovery(cfg); + now += 899_999; + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl]); + expect(hasSpendableCodexCredits(getMainPolicyQuota())).toBe(false); + now++; + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl, whamUrl]); + expect(hasSpendableCodexCredits(getMainPolicyQuota())).toBe(true); +}); + + +test.each([401, 403])("passive option survives identity retry for terminal WHAM %s", async status => { + fetchWith(async () => { + if (calls.length === 1) { writeMain("fixture-replacement"); return usage(); } + return Response.json({ detail: { code: "invalid_workspace_selected" } }, { status }); + }); + const result = await fetchMainAccountInfoAttempt(true, 1, undefined, false, true, false, cfg, { passive: true }); + expect(calls).toEqual([whamUrl, whamUrl]); + expect(result.terminalAuthFailure).toBe(true); + expect(isAccountNeedsReauth(MAIN)).toBe(false); +}); + +test.each([true, false])("passive successful probe retains reauth marked in flight (identity retry=%j)", async retry => { + fetchWith(async () => { + if (retry && calls.length === 1) { writeMain("fixture-replacement"); return usage(); } + markAccountNeedsReauth(MAIN); + return usage(); + }); + await fetchMainAccountInfoAttempt(true, 1, undefined, false, true, false, cfg, { passive: true }); + expect(calls).toHaveLength(retry ? 2 : 1); + expect(isAccountNeedsReauth(MAIN)).toBe(true); +}); + + +test.each([401, 403])("terminal passive WHAM %s retains recovery backoff without quarantine", async status => { + fetchWith(async () => Response.json({ detail: { code: "invalid_workspace_selected" } }, { status })); + await runMainAccountHardLockRecovery(cfg); + now += 60_000; + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + now += 240_000; + await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([whamUrl, whamUrl]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); +}); + + +async function joinPassiveRenewal(explicitRefresh: boolean) { + const entered = deferred(); + const joined = deferred(); + const finish = deferred(); + const originalQuery = quotaQueries.fetchCodexUsage; + let queries = 0; + spyOn(quotaQueries, "fetchCodexUsage").mockImplementation((...args: Parameters) => { + const read = originalQuery(...args); + if (++queries === 2) joined.resolve(); + return read; + }); + fetchWith(async () => { entered.resolve(); return finish.promise; }); + const renewal = runMainAccountHardLockRecovery(cfg); + await entered.promise; + const caller = fetchMainAccountInfoAttempt(true, 1, undefined, false, explicitRefresh, false, cfg); + await joined.promise; // The real single-flight join is pending before releasing WHAM. + expect(calls).toEqual([whamUrl]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + return { renewal, caller, finish }; +} + +test.each([ + { status: 401, explicitRefresh: true }, { status: 403, explicitRefresh: true }, + { status: 401, explicitRefresh: false }, { status: 403, explicitRefresh: false }, +])("ordinary caller joining passive renewal quarantines terminal WHAM %j", async ({ status, explicitRefresh }) => { + const { renewal, caller, finish } = await joinPassiveRenewal(explicitRefresh); + setMainAccountInfoCache({ email: null, plan: "plus", quota: null, ts: now }); + finish.resolve(Response.json({ detail: { code: "invalid_workspace_selected" } }, { status })); + const [, result] = await Promise.all([renewal, caller]); + expect(calls).toEqual([whamUrl]); + expect(result.terminalAuthFailure).toBe(true); + expect(result.quotaRefresh).toEqual({ status: "http_error", httpStatus: status, code: "invalid_workspace_selected" }); + expect(isAccountNeedsReauth(MAIN)).toBe(true); + expect(getMainAccountInfoCache()).toBeNull(); + expect(isMainAccountIdentityGenerationLive(result.quotaRefreshGeneration!)).toBe(true); + expect(getNativeMainProfileRequestCount()).toBe(0); +}); + +test.each([true, false])("successful caller joining passive renewal clears quarantine only for explicit refresh=%j", async explicitRefresh => { + const { renewal, caller, finish } = await joinPassiveRenewal(explicitRefresh); + markAccountNeedsReauth(MAIN); + finish.resolve(usage()); + const [, result] = await Promise.all([renewal, caller]); + expect(calls).toEqual([whamUrl]); + expect(result.quotaRefresh?.status).toBe("ok"); + expect(isAccountNeedsReauth(MAIN)).toBe(!explicitRefresh); + expect(getNativeMainProfileRequestCount()).toBe(0); +}); + +test.each(["credential", "configuration"])("joined terminal-auth evidence cannot quarantine after %s changes", async fence => { + const { renewal, caller, finish } = await joinPassiveRenewal(true); + if (fence === "credential") writeMain(accountId, true); + else { + const generation = sweeper.captureConfigGeneration(); + spyOn(sweeper, "captureConfigGeneration").mockReturnValue(generation + 1); + } + finish.resolve(Response.json({ detail: { code: "invalid_workspace_selected" } }, { status: 403 })); + const [, result] = await Promise.all([renewal, caller]); + expect(calls).toEqual([whamUrl]); + expect(result.terminalAuthFailure).toBeUndefined(); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + expect(getNativeMainProfileRequestCount()).toBe(0); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 700841acd0a..a9ebc6e5cd1 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -683,7 +683,7 @@ "main-account-hard-lock-auth.test.ts": "codex-integration", "main-account-hard-lock-default.test.ts": "codex-integration", "main-account-hard-lock-policy.test.ts": "codex-integration", - "main-account-hard-lock-recovery.test.ts": "codex-integration", + "main-account-hard-lock-recovery.test.ts": "codex-integration", "main-account-credit-renewal.test.ts": "codex-integration", "main-account-policy-binding-wait.test.ts": "codex-integration", "main-device-reauth-api.test.ts": "codex-integration", "main-device-reauth-ui.test.ts": "gui", "main-device-reauth.test.ts": "codex-integration", "main-quota-evidence-validation.test.ts": "codex-integration",