From 4540989f2f76ab36bb1bf7dac04f60f33a853323 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:17:15 -0700 Subject: [PATCH 1/3] fix(cli): a possessive weekly or per-model limit is QUOTA, not RUNTIME MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refresh of this branch's August work, which sat 83 commits behind main with cli/package.json conflicting. The classifier change is theirs; the measurement, the comment rewrite and the test changes below are new. Measured on main 54eb2abc8 before touching anything: "You've hit your weekly limit", the same wording with a model and reset time, "You've reached your limit for claude-opus-4.5" and "you've hit your 5-hour limit" all classify RUNTIME. RUNTIME is the fallthrough class with the shortest ladder (5s/10s/60s), so the fleet probes a quota-blocked seat hardest and burns all three requeue attempts inside 90 seconds. QUOTA_RE gains `weekly limit`; QUOTA_POSSESSIVE_RE matches the sentence shape "you've hit/reached your ... limit" with a negative lookahead that keeps rate limits out, since QUOTA is tested first. The dot carve-out `\.(?=\d)` lets dotted model names through, which plain `[^.\n]` silently excluded. sprint-review's gate on the original head found one test vacuous: "does not let the dot carve-out span a sentence boundary" passed identically with and without the carve-out, because a letter follows the period there. It is renamed to the property the pattern actually has — a period followed by a SPACE ends the match — and the weaker bound it does not have is now pinned explicitly as a known, accepted limit, so it cannot drift unnoticed. Added from the same gate: a capitalised "Rate Limit" case, since the lookahead's `i` is load-bearing and invisible, and a case for the {0,40} length bound. cli 0.1.122 -> 0.1.126 for the publish guard: 0.1.123, 0.1.124 and 0.1.125 are claimed by the open #2412, #2445 and #2446. Co-Authored-By: Claude Opus 5 Co-authored-by: nova Agent-Model: claude/claude-opus-5/high Agent-Model: claude/claude-haiku-5-5/default --- cli/__tests__/spawn-retry.test.mjs | 128 +++++++++++++++++++++++++++++ cli/src/lib/spawn-retry.js | 36 +++++++- 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 599bd097df..346b1cecc0 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -173,3 +173,131 @@ describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () = .toBe(SPAWN_FAILURE_CLASS.RUNTIME); }); }); + +describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', () => { + // Verbatim from the fleet's own logs, measured 2026-08-25 across every seat + // that had failed at all. Three wordings, one sentence shape, and the + // allowlist carried exactly one of them: + // + // 283 × "You've hit your weekly limit" <- unmatched, RUNTIME ladder + // 55 × "You've hit your session limit" <- matched (the 08-18 fix) + // 6 × "You've reached your Fable 5 limit" <- unmatched, RUNTIME ladder + // + // Re-measured on main 54eb2abc8 before this change: all four wordings in this + // file's QUOTA cases below still classified RUNTIME, so the gap is live. + const WEEKLY = "claude exited with code 1: You've hit your weekly limit " + + '· resets Aug 27 at 2pm (America/Los_Angeles)'; + const PER_MODEL = "claude exited with code 1: You've reached your Fable 5 limit"; + + test.each([ + ['weekly, the wording behind 283 of 365 fleet failures', WEEKLY], + ['per-model, whose variable part is a model name', PER_MODEL], + ])('%s is QUOTA, not RUNTIME', (_label, message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('so the circuit opens at the full cooldown on the FIRST failure', () => { + const { circuitOpen, delayMs } = spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(circuitOpen).toBe(true); + expect(delayMs).toBe(SPAWN_RETRY_MAX_MS); + }); + + // This is the specific harm the misclassification caused, and it is not the + // latency. The RUNTIME ladder is 5s / 10s / 60s, so three consecutive + // failures land inside ~75 seconds — and the event is left unacked on each + // (agent.js, "the kernel must retain the event for at-least-once delivery"), + // which burns all three requeue attempts and retires it to `status: 'failed'`. + // Under QUOTA the first failure alone opens a 15-minute circuit. + test('the RUNTIME ladder would burn all three requeue attempts inside 90s', () => { + const runtimeLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error('claude exited with code 1: something transient'), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(runtimeLadder.reduce((a, b) => a + b, 0)).toBeLessThan(90 * 1000); + + const quotaLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(quotaLadder.every((ms) => ms === SPAWN_RETRY_MAX_MS)).toBe(true); + }); + + // Guard on the widening. The phrase requires "your", so a server-side + // throttle phrased the same way must not be swallowed into the 15-minute + // cooldown — QUOTA is tested before RATE_LIMIT, so the ordering is load-bearing. + test.each([ + "You've hit your rate limit, retry shortly", + "You've reached your rate-limit for this model", + // Capitalised, because the lookahead's `i` is load-bearing and invisible. + "you've hit your Rate Limit", + ])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + test('the phrase does not span a sentence boundary', () => { + expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit."))) + .toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); + + describe('the dotted-model-name gap (sprint-review, 2026-08-25)', () => { + // `[^.\n]` was chosen to keep the match inside one sentence, and silently + // also excluded every model name we run, because they are all dotted. + // RUNTIME is the fallthrough with the SHORTEST backoff, so the net effect + // was to probe a quota-blocked seat hardest. + test.each([ + ["You've reached your Haiku 4.5 limit"], + ["you've hit your gpt-5.4-mini limit"], + ["Error: You've reached your Claude Opus 4.8 limit for today"], + ])('classifies %s as QUOTA', (text) => { + expect(classifySpawnFailure(new Error(text))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('still refuses a rate limit, which QUOTA is tested before', () => { + expect(classifySpawnFailure(new Error("you've hit your rate limit"))) + .toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + // RENAMED from "does not let the dot carve-out span a sentence boundary", + // which sprint-review's gate on #1215 showed was vacuous: it passed + // identically with and without the carve-out, because a LETTER follows the + // period there and plain `[^.\n]` already excluded that. What the pattern + // actually has is weaker, and this is it — the space after the period is + // what ends the match. + test('a period followed by a SPACE still terminates the match', () => { + expect(classifySpawnFailure(new Error("You've reached your goal. 5 things have no limit"))) + .not.toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + // The documented limit, asserted so it cannot drift unnoticed: a digit + // immediately after the period IS admitted, even at a sentence boundary. + // Kept rather than tightened — adjacent `.`+digit is rare in provider copy, + // and the alternative predicate is harder to read than the bug is costly. + test('a period followed immediately by a DIGIT is admitted — known and accepted', () => { + expect(classifySpawnFailure(new Error("You've reached your goal.5 things have no limit"))) + .toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + // The {0,40} bound counts alternation units, so a long model name falls + // through to RUNTIME by design. Pinned because nothing else pins it. + test('past the length bound it falls through to RUNTIME', () => { + expect(classifySpawnFailure(new Error( + "You've reached your extremely verbose vendor model name 4.5 preview limit", + ))).toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); + + test('a dotted model name costs the quota ladder, not the runtime ladder', () => { + const policy = spawnRetryPolicy({ + error: new Error("You've reached your Haiku 4.5 limit"), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS }); + }); + }); +}); diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 4824b00854..50b27b43b6 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -27,6 +27,7 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // outage has already been misclassified. Twice now: // 2026-08-03 codex "Your workspace is out of credits." → `out of credits` // 2026-08-18 claude "You've hit your session limit" → `session limit` +// 2026-08-25 claude "You've hit your weekly limit" → the phrase below // The second one is the instructive failure: `usage limit` was already here — // it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on // a string one word away from a pattern we had. Both times the miss meant @@ -62,7 +63,38 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // nothing else until the kernel retired the event. Only the refusal wordings // themselves are matched, so an ordinary failure that mentions a flag is not. const CONTENT_REFUSED_RE = /(?:flagged for possible cybersecurity risk|this content was flagged|content[_ ]policy[_ ]violation|cyber[_ ]policy)/i; -const QUOTA_RE = /(?:quota|usage limit|session limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +// The third miss is why there is now a PHRASE as well as a word list, and it is +// a different failure from the first two. Measured across the fleet's logs on +// 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading +// "You've hit your session limit" (matched), and 6 reading "You've reached your +// Fable 5 limit" (not matched). Three wordings, one sentence shape, and the list +// had caught exactly one of them. Enumerating per-wording keeps failing because +// the variable part is a BILLING PERIOD or a MODEL NAME, both of which the +// providers keep adding, so a list is structurally one release behind. +// +// `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must +// be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is +// what distinguishes exhaustion from a server-side throttle. The negative +// lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and +// without it that string would take the 15-minute cooldown instead of the +// 60-second rate-limit backoff. The length bound stops it spanning sentences. +// +// The character class admits a dot ONLY when a digit follows it, and that +// carve-out is load-bearing rather than cosmetic (sprint-review, 2026-08-25): +// a plain `[^.\n]` reads as "stay inside one sentence" and silently also means +// "no model names". Every model we run is dotted, so "you've reached your Haiku +// 4.5 limit" fell through to RUNTIME, the shortest backoff — the class excluded +// the exact variable part this paragraph says keeps changing. +// +// Known limit of `\.(?=\d)`, kept rather than papered over: a period with a +// digit immediately after it is admitted even at a sentence boundary, so +// "You've reached your goal.5 things have no limit" classes as QUOTA. The space +// in ordinary prose ("goal. 5 things") is what terminates the match, and +// adjacent `.`+digit is rare enough in provider copy to accept. The test below +// pins what the pattern ACTUALLY has — a space after the period ends the match — +// rather than the stronger property it does not have. +const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; @@ -84,7 +116,7 @@ export const classifySpawnFailure = (error) => { // Provider APIs commonly report an exhausted account quota as HTTP 429. // Prefer the more specific body/message over the generic status code so a // hard quota failure gets the long cooldown rather than a one-minute probe. - if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; + if (QUOTA_RE.test(text) || QUOTA_POSSESSIVE_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; if (error?.status === 429 || RATE_LIMIT_RE.test(text)) { return SPAWN_FAILURE_CLASS.RATE_LIMIT; } From 4e398687be284c9de22a5f0d5834624a5fc5eff2 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 23 Oct 2026 01:32:59 -0700 Subject: [PATCH 2/3] fix(cli): a throttle wording anywhere beats the possessive quota shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otto's gate on #2491 found a regression I introduced, not a gap I inherited. `QUOTA_POSSESSIVE_RE`'s `(?!rate[ -]?limit)` lookahead only guards the word IMMEDIATELY after "your", so "You've hit your API rate limit" matched the possessive shape and took QUOTA's 15-minute circuit. Main classifies the same string RATE_LIMIT and probes in 60 seconds. Verified before fixing: on this branch's previous head that string gave `quota` and on main it gives `rate_limit`. The call site now refuses the possessive shape when RATE_LIMIT_RE matches anywhere in the text, reusing the one list of throttle wordings rather than widening the lookahead. QUOTA_RE is still tested first, so an explicit quota word keeps the long cooldown even when the same sentence mentions rate limits — pinned, since that is this fix's own boundary. Three cases red without the guard: "your API rate limit", "your per-model rate limit", "your organisation's rate limit for this model". Co-Authored-By: Claude Opus 5 Co-authored-by: nova Agent-Model: claude/claude-opus-5/high Agent-Model: claude/claude-haiku-5-5/default --- cli/__tests__/spawn-retry.test.mjs | 18 ++++++++++++++++++ cli/src/lib/spawn-retry.js | 13 ++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 346b1cecc0..676f8b603a 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -236,10 +236,28 @@ describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', "You've reached your rate-limit for this model", // Capitalised, because the lookahead's `i` is load-bearing and invisible. "you've hit your Rate Limit", + // The lookahead only guards the word IMMEDIATELY after "your", so these + // two walked past it into the 15-minute circuit while main correctly gave + // the 60-second probe — a regression this branch introduced and otto's + // gate on #2491 caught. The call site now refuses the possessive shape + // whenever a throttle wording appears anywhere in the text. + "You've hit your API rate limit", + "You've reached your per-model rate limit", + "You've hit your organisation's rate limit for this model", ])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => { expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); }); + // The boundary of that refusal, in the other direction: an explicit quota + // WORD still wins, because QUOTA_RE is tested before the possessive shape. + // Otherwise a provider sentence that names both would lose the long cooldown + // it needs. + test('an explicit quota word beats a rate-limit mention in the same message', () => { + expect(classifySpawnFailure(new Error( + 'Claude usage limit reached; standard rate limits also apply.', + ))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + test('the phrase does not span a sentence boundary', () => { expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit."))) .toBe(SPAWN_FAILURE_CLASS.RUNTIME); diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 50b27b43b6..79907af221 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -116,7 +116,18 @@ export const classifySpawnFailure = (error) => { // Provider APIs commonly report an exhausted account quota as HTTP 429. // Prefer the more specific body/message over the generic status code so a // hard quota failure gets the long cooldown rather than a one-minute probe. - if (QUOTA_RE.test(text) || QUOTA_POSSESSIVE_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; + if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; + // The possessive shape is deliberately loose about what sits between "your" + // and "limit", and its `(?!rate[ -]?limit)` lookahead only guards the word + // IMMEDIATELY after "your" — so "you've hit your API rate limit" walked past + // it into the 15-minute circuit, where main correctly gives the 60-second + // rate-limit probe (otto, gate on #2491; it was a regression, not a gap). + // A throttle wording anywhere in the text therefore wins over the shape. + // QUOTA_RE is still tested first, so an explicit quota word keeps QUOTA even + // when the same message also mentions rate limits. + if (QUOTA_POSSESSIVE_RE.test(text) && !RATE_LIMIT_RE.test(text)) { + return SPAWN_FAILURE_CLASS.QUOTA; + } if (error?.status === 429 || RATE_LIMIT_RE.test(text)) { return SPAWN_FAILURE_CLASS.RATE_LIMIT; } From 182ac2a4b77e2bb7dace18ea9052c93d2176676e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:05:35 -0700 Subject: [PATCH 3/3] fix(cli): a 429 status prefix is not a throttle wording (otto HOLD 5638173449) '429: You've reached your Fable 5 limit' matched the 429 token in RATE_LIMIT_RE, which the throttle-wins override used, so an exhausted allowance took the 60s RATE_LIMIT probe. The override now checks throttle wording only; the final status check still classes a bare 429 as RATE_LIMIT. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: nova Agent-Model: claude/claude-haiku-5-5/default --- cli/__tests__/spawn-retry.test.mjs | 3 +++ cli/src/lib/spawn-retry.js | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 676f8b603a..bf0564bec3 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -192,6 +192,9 @@ describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', test.each([ ['weekly, the wording behind 283 of 365 fleet failures', WEEKLY], ['per-model, whose variable part is a model name', PER_MODEL], + // The HTTP status is a prefix here, not a throttle wording. otto's HOLD on + // #2553 (review 5638173449): this shape classed RUNTIME_LIMIT's 60s probe. + ['per-model behind an HTTP 429 prefix', "429: You've reached your Fable 5 limit"], ])('%s is QUOTA, not RUNTIME', (_label, message) => { expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA); }); diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 79907af221..4f87898c1a 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -95,6 +95,9 @@ const CONTENT_REFUSED_RE = /(?:flagged for possible cybersecurity risk|this cont // rather than the stronger property it does not have. const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i; +// The status code is not a throttle wording: "429: You've reached your Fable 5 +// limit" is an exhausted allowance that a provider reports as HTTP 429. +const THROTTLE_WORDING_RE = /(?:rate[ -]?limit|too many requests|overloaded|capacity)/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; @@ -125,7 +128,7 @@ export const classifySpawnFailure = (error) => { // A throttle wording anywhere in the text therefore wins over the shape. // QUOTA_RE is still tested first, so an explicit quota word keeps QUOTA even // when the same message also mentions rate limits. - if (QUOTA_POSSESSIVE_RE.test(text) && !RATE_LIMIT_RE.test(text)) { + if (QUOTA_POSSESSIVE_RE.test(text) && !THROTTLE_WORDING_RE.test(text)) { return SPAWN_FAILURE_CLASS.QUOTA; } if (error?.status === 429 || RATE_LIMIT_RE.test(text)) {