From 002d8b4a3b38bd7802e423ae2b32806d682e8026 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:40:41 -0700 Subject: [PATCH 1/2] fix(cli): probe quota after stated reset (TASK-380) Co-authored-by: kai Agent-Model: codex/gpt-6-sol/medium Agent-Model: codex/gpt-6.1-sol/medium --- cli/__tests__/run-loop.test.mjs | 41 ++++++++++++ cli/__tests__/spawn-retry.test.mjs | 102 ++++++++++++++++++++++++++--- cli/src/lib/spawn-retry.js | 53 ++++++++++++++- 3 files changed, 185 insertions(+), 11 deletions(-) diff --git a/cli/__tests__/run-loop.test.mjs b/cli/__tests__/run-loop.test.mjs index 9753b978ba..63da6ddd98 100644 --- a/cli/__tests__/run-loop.test.mjs +++ b/cli/__tests__/run-loop.test.mjs @@ -1546,6 +1546,47 @@ describe('performRun', () => { expect(errors[0]).toMatchObject({ failureClass: 'quota', circuitOpen: true }); }); + test('a Claude reset hint schedules the next batch after reset, not every 15 minutes', async () => { + const now = Date.parse('2026-10-23T09:57:37Z'); + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const event = makeEvent({ _id: 'evt-quota-reset' }); + createClient.mockReturnValue({ + get: jest.fn().mockResolvedValue({ events: [event] }), + post: jest.fn().mockResolvedValue({}), + }); + const spawn = jest.fn().mockRejectedValue(new Error( + "claude exited with code 1: You've hit your session limit " + + '· resets 4:20am (America/Los_Angeles)', + )); + const scheduled = []; + const errors = []; + const handle = performRun({ + instanceUrl: 'http://localhost:5000', + token: 'cm_agent_test', + adapter: { name: 'stub', detect: stubAdapter.detect, spawn }, + agentName: 'my-stub', + retryJitterRatio: 0, + setTimeoutImpl: (callback, delayMs) => { + scheduled.push({ callback, delayMs }); + return 0; + }, + onError: (error) => errors.push(error), + }); + await drainMicrotasks(); + handle.stop(); + + const delayMs = Date.parse('2026-10-23T11:22:00Z') - now; + expect(spawn).toHaveBeenCalledTimes(1); + expect(scheduled[0].delayMs).toBe(delayMs); + expect(errors[0]).toMatchObject({ + failureClass: 'quota', circuitOpen: true, retryAfterMs: delayMs, + }); + } finally { + clock.mockRestore(); + } + }); + test('a batch spawn failure leaves its no-op sibling unacked for coherent retry', async () => { let poll = 0; const mockGet = jest.fn(async (route) => { diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index bf0564bec3..b34f04e0f0 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -1,4 +1,4 @@ -import { describe, expect, test } from '@jest/globals'; +import { describe, expect, jest, test } from '@jest/globals'; import { SPAWN_FAILURE_CLASS, @@ -28,6 +28,84 @@ describe('spawn retry policy', () => { } }); + test('a named quota reset holds the first probe until after the reset', () => { + const now = Date.parse('2026-10-23T09:57:37Z'); // 2:57:37am in Los Angeles + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const error = new Error( + "claude exited with code 1: You've hit your session limit " + + '· resets 4:20am (America/Los_Angeles)', + ); + const policy = spawnRetryPolicy({ error, consecutiveFailures: 1, intervalMs: 5000 }); + expect(policy).toEqual({ + failureClass: SPAWN_FAILURE_CLASS.QUOTA, + circuitOpen: true, + delayMs: Date.parse('2026-10-23T11:22:00Z') - now, + }); + expect(spawnRetryPolicy({ + error, consecutiveFailures: 2, intervalMs: 5000, jitterRatio: 0.2, + }).delayMs).toBe(policy.delayMs + 24_000); + } finally { + clock.mockRestore(); + } + }); + + test('a reset clock across midnight still names the next local day', () => { + const now = Date.parse('2026-10-23T06:55:00Z'); // 11:55pm in Los Angeles + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 12:10am (America/Los_Angeles)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy.delayMs).toBe(17 * 60 * 1000); + } finally { + clock.mockRestore(); + } + }); + + test.each([ + ['spring forward', '2026-03-08T09:30:00Z', '2026-03-08T11:22:00Z'], + ['fall back', '2026-11-01T07:30:00Z', '2026-11-01T12:22:00Z'], + ])('a quota reset across the %s clock change uses the named zone', (_, nowText, probeText) => { + const now = Date.parse(nowText); + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 4:20am (America/Los_Angeles)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy.delayMs).toBe(Date.parse(probeText) - now); + } finally { + clock.mockRestore(); + } + }); + + test('an invalid reset hint keeps the bounded quota fallback', () => { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 25:99am (America/Los_Angeles)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy.delayMs).toBe(SPAWN_RETRY_MAX_MS); + }); + + test('a just-passed reset clock does not defer the next probe for a day', () => { + const clock = jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-10-23T11:21:00Z')); + try { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 4:20am (America/Los_Angeles)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy.delayMs).toBe(SPAWN_RETRY_MAX_MS); + } finally { + clock.mockRestore(); + } + }); + test('stable agent jitter is bounded and separates fleet probes', () => { const alpha = spawnRetryJitter('alpha'); const beta = spawnRetryJitter('beta'); @@ -114,14 +192,20 @@ describe('classifies the Claude CLI session-limit string (2026-08-18 outage)', ( .toBe(SPAWN_FAILURE_CLASS.QUOTA); }); - test('so it opens the circuit at the full cooldown on the first failure', () => { - const { circuitOpen, delayMs } = spawnRetryPolicy({ - error: new Error(SESSION_LIMIT), - consecutiveFailures: 1, - intervalMs: 5000, - }); - expect(circuitOpen).toBe(true); - expect(delayMs).toBe(SPAWN_RETRY_MAX_MS); + test('so it opens the circuit until the stated reset on the first failure (TASK-380)', () => { + const now = Date.parse('2026-08-18T10:00:00Z'); // 3am in Los Angeles + const clock = jest.spyOn(Date, 'now').mockReturnValue(now); + try { + const { circuitOpen, delayMs } = spawnRetryPolicy({ + error: new Error(SESSION_LIMIT), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(circuitOpen).toBe(true); + expect(delayMs).toBe(62 * 60 * 1000); + } finally { + clock.mockRestore(); + } }); // Guard on the widening, not just the fix. QUOTA is tested before diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 4f87898c1a..5260b626c5 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -19,7 +19,7 @@ export const SPAWN_FAILURE_CLASS = Object.freeze({ }); export const SPAWN_CIRCUIT_THRESHOLD = 3; -// Base ceiling before the stable per-agent 0–20% anti-herd offset. +// Fallback ceiling when an error has no usable reset clock. export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000; export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; @@ -100,6 +100,8 @@ const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[ const THROTTLE_WORDING_RE = /(?:rate[ -]?limit|too many requests|overloaded|capacity)/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; +const QUOTA_RESET_RE = /\bresets?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(([A-Za-z_]+(?:\/[A-Za-z_]+)+)\)/i; +const QUOTA_RESET_MARGIN_MS = 2 * 60 * 1000; const errorText = (error) => [ error?.message, @@ -164,6 +166,46 @@ const applyJitter = (delayMs, jitterRatio) => { return Math.round(delayMs * (1 + safeJitter)); }; +// Claude names the local reset clock in its quota error. Waiting for that +// clock avoids repeatedly launching a seat that cannot answer before then. +// If the hint is absent or malformed, keep the ordinary bounded retry path. +const quotaResetDelay = (error, now = Date.now()) => { + const match = QUOTA_RESET_RE.exec(errorText(error)); + if (!match) return null; + const hour = Number(match[1]); + const minute = Number(match[2] || 0); + if (hour < 1 || hour > 12 || minute > 59) return null; + const target = ((hour % 12) + (match[3].toLowerCase() === 'pm' ? 12 : 0)) * 60 + minute; + let formatter; + try { + formatter = new Intl.DateTimeFormat('en-US', { + timeZone: match[4], hour: '2-digit', minute: '2-digit', hourCycle: 'h23', + }); + } catch { + return null; + } + const localMinute = (instant) => { + const parts = Object.fromEntries(formatter.formatToParts(instant).map(({ type, value }) => [type, value])); + return Number(parts.hour) * 60 + Number(parts.minute); + }; + const nowMinute = localMinute(now); + const wallMinutes = (target - nowMinute + 1440) % 1440; + // At or just after the named minute, the message may be a stale failure + // from the old quota window. Probe on the ordinary cooldown instead of + // assuming the next reset is nearly 24 hours away. + if (wallMinutes === 0 || wallMinutes > 1430) return null; + let resetAt = now - (now % 60000) + wallMinutes * 60 * 1000; + // Crossing a daylight-saving boundary can shift the initial wall-clock + // estimate by an hour. Reconcile against the target in the named zone. + for (let attempt = 0; attempt < 2; attempt += 1) { + const difference = (target - localMinute(resetAt) + 1440) % 1440; + if (difference === 0) break; + resetAt += (difference > 720 ? difference - 1440 : difference) * 60 * 1000; + } + if (resetAt <= now || localMinute(resetAt) !== target) return null; + return resetAt - now + QUOTA_RESET_MARGIN_MS; +}; + /** * Return the next probe delay and whether the circuit is open. * @@ -187,10 +229,17 @@ export const spawnRetryPolicy = ({ failureClass === SPAWN_FAILURE_CLASS.QUOTA || failureClass === SPAWN_FAILURE_CLASS.CONFIGURATION ) { + const resetDelay = failureClass === SPAWN_FAILURE_CLASS.QUOTA + ? quotaResetDelay(error) + : null; return { failureClass, circuitOpen: true, - delayMs: applyJitter(SPAWN_RETRY_MAX_MS, jitterRatio), + // Spread the reset probe by at most 24 seconds, not a percentage of + // the entire wait: a six-hour hint must not add an hour of lag. + delayMs: resetDelay === null + ? applyJitter(SPAWN_RETRY_MAX_MS, jitterRatio) + : resetDelay + applyJitter(QUOTA_RESET_MARGIN_MS, jitterRatio) - QUOTA_RESET_MARGIN_MS, }; } From bb8d65c89d478dc16ee8f6a09f70918a6f59140e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:25:16 -0700 Subject: [PATCH 2/2] test(cli): cover quota reset fallback and single timer Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- cli/__tests__/run-loop.test.mjs | 1 + cli/__tests__/spawn-retry.test.mjs | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/cli/__tests__/run-loop.test.mjs b/cli/__tests__/run-loop.test.mjs index 63da6ddd98..da1d22b34f 100644 --- a/cli/__tests__/run-loop.test.mjs +++ b/cli/__tests__/run-loop.test.mjs @@ -1578,6 +1578,7 @@ describe('performRun', () => { const delayMs = Date.parse('2026-10-23T11:22:00Z') - now; expect(spawn).toHaveBeenCalledTimes(1); + expect(scheduled).toHaveLength(1); expect(scheduled[0].delayMs).toBe(delayMs); expect(errors[0]).toMatchObject({ failureClass: 'quota', circuitOpen: true, retryAfterMs: delayMs, diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index b34f04e0f0..dff30fd6c4 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -92,6 +92,29 @@ describe('spawn retry policy', () => { expect(policy.delayMs).toBe(SPAWN_RETRY_MAX_MS); }); + test('an unsupported reset zone keeps the bounded quota fallback', () => { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 4:20am (Mars/Olympus)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS }); + }); + + test('a 12pm reset names local noon', () => { + const clock = jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-10-23T18:30:00Z')); + try { + const policy = spawnRetryPolicy({ + error: new Error('session limit · resets 12pm (America/Los_Angeles)'), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy.delayMs).toBe(32 * 60 * 1000); + } finally { + clock.mockRestore(); + } + }); + test('a just-passed reset clock does not defer the next probe for a day', () => { const clock = jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-10-23T11:21:00Z')); try {