diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 599bd097df..676f8b603a 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -173,3 +173,149 @@ describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () = .toBe(SPAWN_FAILURE_CLASS.RUNTIME); }); }); + +describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', () => { + // Verbatim from the fleet's own logs, measured 2026-08-25 across every seat + // that had failed at all. Three wordings, one sentence shape, and the + // allowlist carried exactly one of them: + // + // 283 × "You've hit your weekly limit" <- unmatched, RUNTIME ladder + // 55 × "You've hit your session limit" <- matched (the 08-18 fix) + // 6 × "You've reached your Fable 5 limit" <- unmatched, RUNTIME ladder + // + // Re-measured on main 54eb2abc8 before this change: all four wordings in this + // file's QUOTA cases below still classified RUNTIME, so the gap is live. + const WEEKLY = "claude exited with code 1: You've hit your weekly limit " + + '· resets Aug 27 at 2pm (America/Los_Angeles)'; + const PER_MODEL = "claude exited with code 1: You've reached your Fable 5 limit"; + + test.each([ + ['weekly, the wording behind 283 of 365 fleet failures', WEEKLY], + ['per-model, whose variable part is a model name', PER_MODEL], + ])('%s is QUOTA, not RUNTIME', (_label, message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('so the circuit opens at the full cooldown on the FIRST failure', () => { + const { circuitOpen, delayMs } = spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(circuitOpen).toBe(true); + expect(delayMs).toBe(SPAWN_RETRY_MAX_MS); + }); + + // This is the specific harm the misclassification caused, and it is not the + // latency. The RUNTIME ladder is 5s / 10s / 60s, so three consecutive + // failures land inside ~75 seconds — and the event is left unacked on each + // (agent.js, "the kernel must retain the event for at-least-once delivery"), + // which burns all three requeue attempts and retires it to `status: 'failed'`. + // Under QUOTA the first failure alone opens a 15-minute circuit. + test('the RUNTIME ladder would burn all three requeue attempts inside 90s', () => { + const runtimeLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error('claude exited with code 1: something transient'), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(runtimeLadder.reduce((a, b) => a + b, 0)).toBeLessThan(90 * 1000); + + const quotaLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(quotaLadder.every((ms) => ms === SPAWN_RETRY_MAX_MS)).toBe(true); + }); + + // Guard on the widening. The phrase requires "your", so a server-side + // throttle phrased the same way must not be swallowed into the 15-minute + // cooldown — QUOTA is tested before RATE_LIMIT, so the ordering is load-bearing. + test.each([ + "You've hit your rate limit, retry shortly", + "You've reached your rate-limit for this model", + // Capitalised, because the lookahead's `i` is load-bearing and invisible. + "you've hit your Rate Limit", + // The lookahead only guards the word IMMEDIATELY after "your", so these + // two walked past it into the 15-minute circuit while main correctly gave + // the 60-second probe — a regression this branch introduced and otto's + // gate on #2491 caught. The call site now refuses the possessive shape + // whenever a throttle wording appears anywhere in the text. + "You've hit your API rate limit", + "You've reached your per-model rate limit", + "You've hit your organisation's rate limit for this model", + ])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + // The boundary of that refusal, in the other direction: an explicit quota + // WORD still wins, because QUOTA_RE is tested before the possessive shape. + // Otherwise a provider sentence that names both would lose the long cooldown + // it needs. + test('an explicit quota word beats a rate-limit mention in the same message', () => { + expect(classifySpawnFailure(new Error( + 'Claude usage limit reached; standard rate limits also apply.', + ))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('the phrase does not span a sentence boundary', () => { + expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit."))) + .toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); + + describe('the dotted-model-name gap (sprint-review, 2026-08-25)', () => { + // `[^.\n]` was chosen to keep the match inside one sentence, and silently + // also excluded every model name we run, because they are all dotted. + // RUNTIME is the fallthrough with the SHORTEST backoff, so the net effect + // was to probe a quota-blocked seat hardest. + test.each([ + ["You've reached your Haiku 4.5 limit"], + ["you've hit your gpt-5.4-mini limit"], + ["Error: You've reached your Claude Opus 4.8 limit for today"], + ])('classifies %s as QUOTA', (text) => { + expect(classifySpawnFailure(new Error(text))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('still refuses a rate limit, which QUOTA is tested before', () => { + expect(classifySpawnFailure(new Error("you've hit your rate limit"))) + .toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + // RENAMED from "does not let the dot carve-out span a sentence boundary", + // which sprint-review's gate on #1215 showed was vacuous: it passed + // identically with and without the carve-out, because a LETTER follows the + // period there and plain `[^.\n]` already excluded that. What the pattern + // actually has is weaker, and this is it — the space after the period is + // what ends the match. + test('a period followed by a SPACE still terminates the match', () => { + expect(classifySpawnFailure(new Error("You've reached your goal. 5 things have no limit"))) + .not.toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + // The documented limit, asserted so it cannot drift unnoticed: a digit + // immediately after the period IS admitted, even at a sentence boundary. + // Kept rather than tightened — adjacent `.`+digit is rare in provider copy, + // and the alternative predicate is harder to read than the bug is costly. + test('a period followed immediately by a DIGIT is admitted — known and accepted', () => { + expect(classifySpawnFailure(new Error("You've reached your goal.5 things have no limit"))) + .toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + // The {0,40} bound counts alternation units, so a long model name falls + // through to RUNTIME by design. Pinned because nothing else pins it. + test('past the length bound it falls through to RUNTIME', () => { + expect(classifySpawnFailure(new Error( + "You've reached your extremely verbose vendor model name 4.5 preview limit", + ))).toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); + + test('a dotted model name costs the quota ladder, not the runtime ladder', () => { + const policy = spawnRetryPolicy({ + error: new Error("You've reached your Haiku 4.5 limit"), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS }); + }); + }); +}); diff --git a/cli/package-lock.json b/cli/package-lock.json index d52f3c8cf6..924a2c41f5 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -1,12 +1,12 @@ { "name": "@commonlyai/cli", - "version": "0.1.122", + "version": "0.1.130", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@commonlyai/cli", - "version": "0.1.122", + "version": "0.1.130", "license": "Apache-2.0", "dependencies": { "commander": "^12.0.0" diff --git a/cli/package.json b/cli/package.json index 9b2aa00f99..3108d009e9 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@commonlyai/cli", - "version": "0.1.122", + "version": "0.1.130", "license": "Apache-2.0", "description": "The Commonly CLI — connect agents, manage pods, iterate fast", "type": "module", diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 4824b00854..79907af221 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -27,6 +27,7 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // outage has already been misclassified. Twice now: // 2026-08-03 codex "Your workspace is out of credits." → `out of credits` // 2026-08-18 claude "You've hit your session limit" → `session limit` +// 2026-08-25 claude "You've hit your weekly limit" → the phrase below // The second one is the instructive failure: `usage limit` was already here — // it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on // a string one word away from a pattern we had. Both times the miss meant @@ -62,7 +63,38 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // nothing else until the kernel retired the event. Only the refusal wordings // themselves are matched, so an ordinary failure that mentions a flag is not. const CONTENT_REFUSED_RE = /(?:flagged for possible cybersecurity risk|this content was flagged|content[_ ]policy[_ ]violation|cyber[_ ]policy)/i; -const QUOTA_RE = /(?:quota|usage limit|session limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +// The third miss is why there is now a PHRASE as well as a word list, and it is +// a different failure from the first two. Measured across the fleet's logs on +// 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading +// "You've hit your session limit" (matched), and 6 reading "You've reached your +// Fable 5 limit" (not matched). Three wordings, one sentence shape, and the list +// had caught exactly one of them. Enumerating per-wording keeps failing because +// the variable part is a BILLING PERIOD or a MODEL NAME, both of which the +// providers keep adding, so a list is structurally one release behind. +// +// `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must +// be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is +// what distinguishes exhaustion from a server-side throttle. The negative +// lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and +// without it that string would take the 15-minute cooldown instead of the +// 60-second rate-limit backoff. The length bound stops it spanning sentences. +// +// The character class admits a dot ONLY when a digit follows it, and that +// carve-out is load-bearing rather than cosmetic (sprint-review, 2026-08-25): +// a plain `[^.\n]` reads as "stay inside one sentence" and silently also means +// "no model names". Every model we run is dotted, so "you've reached your Haiku +// 4.5 limit" fell through to RUNTIME, the shortest backoff — the class excluded +// the exact variable part this paragraph says keeps changing. +// +// Known limit of `\.(?=\d)`, kept rather than papered over: a period with a +// digit immediately after it is admitted even at a sentence boundary, so +// "You've reached your goal.5 things have no limit" classes as QUOTA. The space +// in ordinary prose ("goal. 5 things") is what terminates the match, and +// adjacent `.`+digit is rare enough in provider copy to accept. The test below +// pins what the pattern ACTUALLY has — a space after the period ends the match — +// rather than the stronger property it does not have. +const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; @@ -85,6 +117,17 @@ export const classifySpawnFailure = (error) => { // Prefer the more specific body/message over the generic status code so a // hard quota failure gets the long cooldown rather than a one-minute probe. if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; + // The possessive shape is deliberately loose about what sits between "your" + // and "limit", and its `(?!rate[ -]?limit)` lookahead only guards the word + // IMMEDIATELY after "your" — so "you've hit your API rate limit" walked past + // it into the 15-minute circuit, where main correctly gives the 60-second + // rate-limit probe (otto, gate on #2491; it was a regression, not a gap). + // A throttle wording anywhere in the text therefore wins over the shape. + // QUOTA_RE is still tested first, so an explicit quota word keeps QUOTA even + // when the same message also mentions rate limits. + if (QUOTA_POSSESSIVE_RE.test(text) && !RATE_LIMIT_RE.test(text)) { + return SPAWN_FAILURE_CLASS.QUOTA; + } if (error?.status === 429 || RATE_LIMIT_RE.test(text)) { return SPAWN_FAILURE_CLASS.RATE_LIMIT; }