From 65b8644335ab2c910898c8e76b7153daecb575fb Mon Sep 17 00:00:00 2001 From: Daniil Pokrovsky Date: Fri, 28 Aug 2026 16:42:49 +0700 Subject: [PATCH] feat: implement DEV-106 - Automatic failover across multiple agent harnesses when usage limits are hit --- docs/code/configuration.md | 24 ++ packages/agent-harness/src/harness-chain.ts | 242 +++++++++++++++ packages/agent-harness/src/index.ts | 18 +- packages/agent-harness/src/registry.ts | 3 + packages/agent-harness/src/resolver.ts | 19 +- .../agent-harness/tests/harness-chain.test.ts | 214 ++++++++++++++ packages/code/.devintern-code/.env.example | 6 + packages/code/.env.example | 6 + packages/code/src/index.ts | 1 + packages/code/src/lib/address-review.ts | 1 + packages/code/src/lib/harness-failover.ts | 277 ++++++++++++++++++ .../code/src/lib/review-polling-acquirer.ts | 4 +- packages/code/src/lib/webhook-queue.ts | 51 ++++ packages/code/src/webhook-server.ts | 210 +++++++++---- packages/code/tests/harness-failover.test.ts | 250 ++++++++++++++++ 15 files changed, 1261 insertions(+), 65 deletions(-) create mode 100644 packages/agent-harness/src/harness-chain.ts create mode 100644 packages/agent-harness/tests/harness-chain.test.ts create mode 100644 packages/code/src/lib/harness-failover.ts create mode 100644 packages/code/tests/harness-failover.test.ts diff --git a/docs/code/configuration.md b/docs/code/configuration.md index 8053730..d196262 100644 --- a/docs/code/configuration.md +++ b/docs/code/configuration.md @@ -365,6 +365,30 @@ Common `AGENT_HARNESS` values include `claude-code`, `opencode`, `codex`, `curso **Qwen note:** Qwen Code accepts a model via its `--model` flag (e.g. `qwen3-coder-plus`) — set it with `AGENT_MODEL`; you can also keep the model in `~/.qwen/settings.json`. +### Failover across multiple harnesses + +`AGENT_HARNESS` accepts a comma-separated, priority-ordered list so the unattended worker keeps processing when an agent hits its usage limit: + +```bash +# .devintern-code/.env +AGENT_HARNESS=claude-code,codex +``` + +The first entry is your preferred harness; later entries are fallbacks in priority order. A single value behaves exactly as before. + +**Failover behavior (worker mode):** + +- At startup every entry is checked against the harness registry and your machine: unknown or not-installed entries produce a clear warning and are skipped, and the effective chain is logged (e.g. `Agent harness: claude-code → codex (failover enabled)`). +- When the active harness reports a usage/rate limit, the worker records its reset window (parsed from the limit output; a 1-hour cooldown applies when no timer is parseable, e.g. monthly spend limits) and switches to the highest-priority harness that still has capacity. Queued and incoming events continue processing on the fallback without losing anything. +- When the primary harness's window elapses, the worker automatically fails back to it and logs the switch. Fallback agents hitting their own limits mid-queue advance the chain again. +- If every harness in the chain is limited at once, the queue pauses exactly as it does for a single harness and resumes when the earliest window ends. +- Failover state (active harness + per-harness windows) persists in the queue database, so restarting the worker resumes on the right harness instead of retrying a still-limited agent. +- Which harness executed each run is recorded in run records, and `/health` on the webhook server reports the active harness, the chain, and open limit windows. + +One-shot runs (`devintern TASK-123`), review helpers, and other non-worker flows always use the first (priority) entry; multi-harness failover there is out of scope. + +**Per-harness overrides inside a list:** `_CLI_PATH` (e.g. `CODEX_CLI_PATH`) resolves per active harness at spawn time. The global `AGENT_CLI_PATH` applies to the first entry only, so a stale global override cannot leak onto a fallback agent. `AGENT_MODEL` applies to whichever harness is active (the string is harness-specific). + ### Model selection Set the model the agent harness runs with using `AGENT_MODEL` in `.devintern-code/.env`: diff --git a/packages/agent-harness/src/harness-chain.ts b/packages/agent-harness/src/harness-chain.ts new file mode 100644 index 0000000..41ac8a9 --- /dev/null +++ b/packages/agent-harness/src/harness-chain.ts @@ -0,0 +1,242 @@ +/** + * Priority-ordered harness chain (`AGENT_HARNESS=claude-code,codex`). + * + * A comma-separated `AGENT_HARNESS` value names a failover chain: the first + * entry is the preferred harness, later entries are fallbacks used when an + * earlier one hits its usage/rate limit. Parsing and resolution live here so + * every consumer (worker failover, readiness probe, one-shot runs) sees the + * same list semantics: + * + * - Entries are split on commas and trimmed; empty entries are dropped. + * - Aliases (e.g. `agy`, deprecated `gemini`) resolve to canonical names. + * - Duplicate canonical names collapse to the first occurrence, preserving + * the requested priority. + * - An empty/unset value defaults to `["claude-code"]`. + * + * Resolution validates each name against the registry, resolves its CLI path + * per harness (so `AGENT_CLI_PATH` cannot leak across harnesses — harness + * specific `_CLI_PATH` overrides and defaults apply instead), and + * checks installability. Unknown or not-installed entries are reported as + * issues for the caller to warn about and skipped rather than fatal, unless + * that would leave an empty chain (then the full list is kept and the spawn + * itself surfaces the real error, matching single-harness behavior). + */ + +import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; +import { getHarnessCliCommand, isHarnessCliAvailable } from "./resolver.js"; +import type { AgentHarness } from "./types.js"; + +/** + * Parse a raw (possibly comma-separated) harness list into canonical names. + * + * Applies registry aliases, drops empty entries, and de-duplicates canonical + * names keeping the first occurrence. Returns `[DEFAULT_HARNESS_NAME]` when + * nothing usable remains. + * + * @param raw - Raw `AGENT_HARNESS` value (already env-resolved), may be undefined. + * @returns Ordered canonical harness names (priority first). + */ +export function parseHarnessList(raw: string | undefined): string[] { + const names: string[] = []; + const seen = new Set(); + for (const part of (raw ?? "").split(",")) { + const requested = part.trim(); + if (!requested) { + continue; + } + const canonical = getHarness(requested)?.name ?? requested; + if (seen.has(canonical)) { + continue; + } + seen.add(canonical); + names.push(canonical); + } + return names.length > 0 ? names : [DEFAULT_HARNESS_NAME]; +} + +/** One ordered, resolved entry of the harness chain. */ +export interface HarnessChainEntry { + /** Canonical registry name (after alias resolution). */ + readonly name: string; + readonly harness: AgentHarness; + /** Resolved CLI command/path for this harness. */ + readonly path: string; + /** Whether the resolved CLI is installed/reachable on this machine. */ + readonly installed: boolean; +} + +/** A chain entry that could not be used, with a warning message. */ +export interface HarnessChainIssue { + /** Name as written in the list (after alias resolution). */ + readonly requested: string; + readonly reason: "unknown" | "not-installed"; + readonly message: string; +} + +/** Result of resolving a harness chain. */ +export interface ResolvedHarnessChain { + /** Ordered usable entries (priority first). */ + readonly entries: HarnessChainEntry[]; + /** Entries dropped from the chain, with warning messages. */ + readonly issues: HarnessChainIssue[]; + /** All canonical names as parsed from the raw value (before dropping). */ + readonly parsed: string[]; + /** True when the raw value listed more than one harness. */ + readonly multiHarness: boolean; +} + +/** Options for {@link resolveHarnessChain}. */ +export interface HarnessChainOptions { + /** Raw list value; defaults to `AGENT_HARNESS` env (or the default harness). */ + raw?: string; + /** + * Probe installability and drop not-installed entries (default true). + * When false, every known entry is kept with `installed` reported as-is. + */ + checkInstalled?: boolean; + /** + * When false, suppress deprecation warnings for aliased harness names + * (e.g. `gemini` → `antigravity`). Defaults to true. + */ + warnDeprecated?: boolean; + /** Installability predicate override (tests). Defaults to PATH probing. */ + isInstalled?: (entry: { name: string; path: string }) => boolean; +} + +/** + * Resolve the CLI path for one parsed chain entry. + * + * Harness-specific `_CLI_PATH` overrides and the harness default + * apply; the global `AGENT_CLI_PATH` only applies to the first (priority) + * entry so a single-value configuration keeps resolving exactly like + * {@link resolveHarness} does today, while a stale global override cannot + * stick to a fallback harness selected during failover. + * + * @param harness - Registered harness for the entry. + * @param isPrimary - Whether this is the first (priority) chain entry. + * @param warnDeprecated - Whether legacy path env vars may warn. + * @returns The CLI command or path to spawn/probe for this entry. + */ +function resolveEntryPath( + harness: AgentHarness, + isPrimary: boolean, + warnDeprecated?: boolean, +): string { + if (isPrimary && process.env.AGENT_CLI_PATH) { + return process.env.AGENT_CLI_PATH; + } + return getHarnessCliCommand(harness, { warnDeprecated }); +} + +/** + * Resolve the priority-ordered harness chain from a comma-separated value. + * + * Unknown names and (when `checkInstalled`) not-installed CLIs are reported in + * `issues` and skipped; if nothing survives, the full parsed list is kept so + * spawning fails with the familiar actionable error instead of an empty chain. + * + * @param options - Raw value override, installability toggles, and test hooks. + * @returns The resolved chain with usable entries and dropped-entry issues. + */ +export function resolveHarnessChain(options: HarnessChainOptions = {}): ResolvedHarnessChain { + const raw = options.raw ?? process.env.AGENT_HARNESS; + const parsed = parseHarnessList(raw); + const checkInstalled = options.checkInstalled ?? true; + const warnDeprecated = options.warnDeprecated; + + const entries: HarnessChainEntry[] = []; + const issues: HarnessChainIssue[] = []; + + const buildEntry = ( + canonical: string, + isPrimary: boolean, + reportIssues: boolean, + enforceInstalled: boolean, + ): HarnessChainEntry | null => { + const harness = getHarness(canonical); + if (!harness) { + if (reportIssues) { + const availableNames = listHarnesses() + .map((h) => `"${h.name}"`) + .join(", "); + issues.push({ + requested: canonical, + reason: "unknown", + message: `Unknown agent harness "${canonical}" in AGENT_HARNESS; skipping it. Available harnesses: ${availableNames}.`, + }); + } + return null; + } + + const path = resolveEntryPath(harness, isPrimary, warnDeprecated); + const installed = options.isInstalled + ? options.isInstalled({ name: harness.name, path }) + : isHarnessCliAvailable(path); + + if (enforceInstalled && checkInstalled && !installed) { + if (reportIssues) { + const envKey = harness.name.toUpperCase().replace(/-/g, "_"); + issues.push({ + requested: canonical, + reason: "not-installed", + message: `Harness "${canonical}" is not installed (looked for "${path}"); skipping it. Install its CLI or set ${envKey}_CLI_PATH to the executable.`, + }); + } + return null; + } + + return { name: harness.name, harness, path, installed }; + }; + + // Emit deprecation warnings for requested alias names (e.g. `gemini`), + // once per distinct alias, mirroring resolveHarness's behavior. + if (warnDeprecated !== false) { + const warnedAliases = new Set(); + for (const token of (raw ?? "").split(",")) { + const requested = token.trim(); + const alias = requested ? HARNESS_ALIASES[requested] : undefined; + if (alias?.deprecated && alias.warning && !warnedAliases.has(requested)) { + warnedAliases.add(requested); + console.warn(`⚠️ ${alias.warning}`); + } + } + } + + parsed.forEach((canonical, index) => { + const entry = buildEntry(canonical, index === 0, true, true); + if (entry) { + entries.push(entry); + } + }); + + if (entries.length === 0) { + if (parsed.every((canonical) => !getHarness(canonical))) { + // Nothing but unknown names: a configuration error, same as handing + // resolveHarness a single invalid name. + const availableNames = listHarnesses() + .map((h) => `"${h.name}"`) + .join(", "); + throw new Error( + `Unknown agent harness "${parsed.join(",")}" in AGENT_HARNESS. ` + + `Available harnesses: ${availableNames}. ` + + `Set AGENT_HARNESS to a comma-separated list of registered harnesses.`, + ); + } + // Everything was dropped as not installed: keep the full parsed list so + // the spawn path reports the familiar "CLI not found" error instead of + // failing startup with an empty chain. + parsed.forEach((canonical, index) => { + const entry = buildEntry(canonical, index === 0, false, false); + if (entry) { + entries.push(entry); + } + }); + } + + return { + entries, + issues, + parsed, + multiHarness: parsed.length > 1, + }; +} diff --git a/packages/agent-harness/src/index.ts b/packages/agent-harness/src/index.ts index 580c07f..36e2ab3 100644 --- a/packages/agent-harness/src/index.ts +++ b/packages/agent-harness/src/index.ts @@ -32,7 +32,23 @@ export { } from "./modes.js"; // Registry -export { registerHarness, getHarness, listHarnesses, HARNESS_ALIASES } from "./registry.js"; +export { + registerHarness, + getHarness, + listHarnesses, + HARNESS_ALIASES, + DEFAULT_HARNESS_NAME, +} from "./registry.js"; + +// Harness chain (comma-separated AGENT_HARNESS failover lists) +export { + parseHarnessList, + resolveHarnessChain, + type HarnessChainEntry, + type HarnessChainIssue, + type HarnessChainOptions, + type ResolvedHarnessChain, +} from "./harness-chain.js"; // Prompt argument construction export { buildPromptArgs } from "./prompt-args.js"; diff --git a/packages/agent-harness/src/registry.ts b/packages/agent-harness/src/registry.ts index 96dd8a5..2f56a2e 100644 --- a/packages/agent-harness/src/registry.ts +++ b/packages/agent-harness/src/registry.ts @@ -22,6 +22,9 @@ import { const registry = new Map(); +/** Default harness when no harness is configured (e.g. `AGENT_HARNESS` unset). */ +export const DEFAULT_HARNESS_NAME = "claude-code"; + /** * Alias map: alternate harness ids → canonical registered name. * diff --git a/packages/agent-harness/src/resolver.ts b/packages/agent-harness/src/resolver.ts index 40b7600..26da541 100644 --- a/packages/agent-harness/src/resolver.ts +++ b/packages/agent-harness/src/resolver.ts @@ -3,7 +3,8 @@ * * Resolution order for harness name: * 1. `options.harnessName` - * 2. `AGENT_HARNESS` environment variable + * 2. `AGENT_HARNESS` environment variable (a comma-separated list uses its + * first entry — see `resolveHarnessChain` for full-chain resolution) * 3. Default to "claude-code" (backward compatible) * * Resolution order for executable path: @@ -17,7 +18,7 @@ import { execSync } from "child_process"; import { accessSync, constants, existsSync } from "fs"; import { resolve } from "path"; -import { getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; +import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js"; import type { AgentHarness, ResolvedHarness } from "./types.js"; export interface HarnessResolutionOptions { @@ -41,6 +42,10 @@ export interface HarnessResolutionOptions { * Resolve which harness to use and the CLI executable path to invoke. * * Harness name: `options.harnessName` → `AGENT_HARNESS` → `"claude-code"`. + * A comma-separated `AGENT_HARNESS` (e.g. `claude-code,codex`) names a + * failover chain; this resolver always picks the first (priority) entry so + * one-shot flows keep behaving as a single-harness run — worker-mode failover + * across the rest of the chain lives in {@link resolveHarnessChain}. * Aliases (e.g. `agy` / deprecated `gemini` → `antigravity`) are applied before * registry lookup; deprecated aliases emit a one-line console warning. * @@ -65,10 +70,16 @@ export function resolveHarness(options?: HarnessResolutionOptions): ResolvedHarn const explicitHarnessName = options?.harnessName; let harnessName = explicitHarnessName; if (!harnessName) { - harnessName = env.AGENT_HARNESS; + // AGENT_HARNESS may name a comma-separated failover chain; env-driven + // (non-explicit) resolution always uses the first (priority) entry, kept + // raw so alias/deprecation warnings still fire for the requested name. + const firstEntry = env.AGENT_HARNESS?.split(",")[0]?.trim(); + if (firstEntry) { + harnessName = firstEntry; + } } if (!harnessName) { - harnessName = "claude-code"; + harnessName = DEFAULT_HARNESS_NAME; } const alias = HARNESS_ALIASES[harnessName]; diff --git a/packages/agent-harness/tests/harness-chain.test.ts b/packages/agent-harness/tests/harness-chain.test.ts new file mode 100644 index 0000000..7fdd826 --- /dev/null +++ b/packages/agent-harness/tests/harness-chain.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { parseHarnessList, resolveHarnessChain } from "../src/harness-chain.js"; +import { resolveHarness } from "../src/resolver.js"; +import { getHarness } from "../src/registry.js"; + +describe("parseHarnessList", () => { + test("defaults to claude-code when raw is undefined", () => { + expect(parseHarnessList(undefined)).toEqual(["claude-code"]); + }); + + test("defaults to claude-code when raw is empty or only separators", () => { + expect(parseHarnessList("")).toEqual(["claude-code"]); + expect(parseHarnessList(" , ,")).toEqual(["claude-code"]); + }); + + test("parses a single name", () => { + expect(parseHarnessList("codex")).toEqual(["codex"]); + }); + + test("splits on commas, trims, and drops empty entries", () => { + expect(parseHarnessList(" claude-code , codex ,, grok ")).toEqual([ + "claude-code", + "codex", + "grok", + ]); + }); + + test("applies aliases to canonical names", () => { + expect(parseHarnessList("agy")).toEqual(["antigravity"]); + expect(parseHarnessList("gemini,codex")).toEqual(["antigravity", "codex"]); + }); + + test("de-duplicates canonical names keeping the first occurrence", () => { + expect(parseHarnessList("codex, codex")).toEqual(["codex"]); + expect(parseHarnessList("gemini, antigravity")).toEqual(["antigravity"]); + expect(parseHarnessList("claude-code, agy, antigravity")).toEqual(["claude-code", "antigravity"]); + }); + + test("keeps unknown names so callers can warn about them", () => { + expect(parseHarnessList("nope")).toEqual(["nope"]); + }); +}); + +describe("resolveHarness with comma-separated AGENT_HARNESS", () => { + const originalEnv = { ...process.env }; + let warnings: string[]; + const originalWarn = console.warn; + + beforeEach(() => { + delete process.env.AGENT_HARNESS; + delete process.env.AGENT_CLI_PATH; + warnings = []; + console.warn = (msg?: unknown) => { + warnings.push(String(msg)); + }; + }); + + afterEach(() => { + console.warn = originalWarn; + for (const key of Object.keys(process.env)) { + if (!(key in originalEnv)) { + delete process.env[key]; + } + } + Object.assign(process.env, originalEnv); + }); + + test("uses the first (priority) entry of the list", () => { + process.env.AGENT_HARNESS = "codex,claude-code"; + const result = resolveHarness(); + expect(result.harness.name).toBe("codex"); + expect(result.path).toBe("codex"); + }); + + test("still warns when the first entry is a deprecated alias", () => { + process.env.AGENT_HARNESS = "gemini,codex"; + const result = resolveHarness(); + expect(result.harness.name).toBe("antigravity"); + expect(warnings.some((w) => w.includes("deprecated"))).toBe(true); + }); + + test("single value behaves exactly as before", () => { + process.env.AGENT_HARNESS = "codex"; + expect(resolveHarness().harness.name).toBe("codex"); + }); +}); + +describe("resolveHarnessChain", () => { + const originalEnv = { ...process.env }; + let warnings: string[]; + const originalWarn = console.warn; + + const alwaysInstalled = () => true; + + beforeEach(() => { + delete process.env.AGENT_HARNESS; + delete process.env.AGENT_CLI_PATH; + warnings = []; + console.warn = (msg?: unknown) => { + warnings.push(String(msg)); + }; + }); + + afterEach(() => { + console.warn = originalWarn; + for (const key of Object.keys(process.env)) { + if (!(key in originalEnv)) { + delete process.env[key]; + } + } + Object.assign(process.env, originalEnv); + }); + + test("returns a single entry for a single-harness value", () => { + const chain = resolveHarnessChain({ raw: "codex", isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex"]); + expect(chain.multiHarness).toBe(false); + expect(chain.issues).toEqual([]); + }); + + test("defaults to claude-code when raw is undefined", () => { + const chain = resolveHarnessChain({ isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["claude-code"]); + expect(chain.parsed).toEqual(["claude-code"]); + }); + + test("keeps priority order and flags multi-harness chains", () => { + const chain = resolveHarnessChain({ + raw: "codex, claude-code , grok", + isInstalled: alwaysInstalled, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "claude-code", "grok"]); + expect(chain.multiHarness).toBe(true); + }); + + test("skips unknown entries with a warning", () => { + const chain = resolveHarnessChain({ + raw: "does-not-exist,codex", + isInstalled: alwaysInstalled, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex"]); + expect(chain.issues).toHaveLength(1); + expect(chain.issues[0]!.reason).toBe("unknown"); + expect(chain.issues[0]!.requested).toBe("does-not-exist"); + expect(chain.issues[0]!.message).toContain("Available harnesses"); + expect(warnings).toEqual([]); + }); + + test("skips not-installed entries with a warning", () => { + const chain = resolveHarnessChain({ + raw: "codex,grok", + isInstalled: ({ name }) => name !== "codex", + }); + expect(chain.entries.map((e) => e.name)).toEqual(["grok"]); + expect(chain.issues).toHaveLength(1); + expect(chain.issues[0]!.reason).toBe("not-installed"); + expect(chain.issues[0]!.message).toContain("CODEX_CLI_PATH"); + }); + + test("throws when every entry is unknown", () => { + expect(() => + resolveHarnessChain({ raw: "nope, also-nope", isInstalled: alwaysInstalled }), + ).toThrow("Unknown agent harness"); + }); + + test("keeps the full list when everything is not installed", () => { + const chain = resolveHarnessChain({ + raw: "codex,grok", + isInstalled: () => false, + }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "grok"]); + expect(chain.entries.every((e) => e.installed === false)).toBe(true); + }); + + test("checkInstalled=false skips installability probing", () => { + const chain = resolveHarnessChain({ raw: "codex,grok", checkInstalled: false }); + expect(chain.entries.map((e) => e.name)).toEqual(["codex", "grok"]); + expect(chain.issues).toEqual([]); + }); + + test("AGENT_CLI_PATH applies to the primary entry only", () => { + process.env.AGENT_CLI_PATH = "/custom/agent"; + const chain = resolveHarnessChain({ raw: "codex,grok", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.path).toBe("/custom/agent"); + expect(chain.entries[1]!.path).toBe("grok"); + }); + + test("harness-specific env overrides resolve per entry", () => { + process.env.GROK_CLI_PATH = "/custom/grok"; + const chain = resolveHarnessChain({ raw: "codex,grok", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.path).toBe("codex"); + expect(chain.entries[1]!.path).toBe("/custom/grok"); + }); + + test("resolves registry harness objects on each entry", () => { + const chain = resolveHarnessChain({ raw: "codex", isInstalled: alwaysInstalled }); + expect(chain.entries[0]!.harness).toBe(getHarness("codex")); + }); + + test("warns once for a deprecated alias inside the list", () => { + const chain = resolveHarnessChain({ raw: "gemini,codex", isInstalled: alwaysInstalled }); + expect(chain.entries.map((e) => e.name)).toEqual(["antigravity", "codex"]); + expect(warnings.filter((w) => w.includes("deprecated"))).toHaveLength(1); + }); + + test("warnDeprecated=false suppresses deprecation warnings", () => { + resolveHarnessChain({ + raw: "gemini", + warnDeprecated: false, + isInstalled: alwaysInstalled, + }); + expect(warnings).toEqual([]); + }); +}); diff --git a/packages/code/.devintern-code/.env.example b/packages/code/.devintern-code/.env.example index 6542394..f07008f 100644 --- a/packages/code/.devintern-code/.env.example +++ b/packages/code/.devintern-code/.env.example @@ -19,6 +19,12 @@ JIRA_API_TOKEN=your-api-token-here # Which AI agent to use: claude-code | opencode | codex | cursor | grok | deepseek # (also: gemini | kimi | qwen | goose | kilo-code | cline | pi) # Defaults to 'claude-code' if not specified +# +# Accepts a comma-separated, priority-ordered list for automatic failover in +# worker mode: when the first harness hits its usage limit, the worker +# switches to the next entry and fails back to the first once its window +# resets. Unknown or not-installed entries are warned about and skipped. +# AGENT_HARNESS=claude-code,codex AGENT_HARNESS=claude-code # Optional: Path to the agent CLI executable. diff --git a/packages/code/.env.example b/packages/code/.env.example index fcfe3a2..d189048 100644 --- a/packages/code/.env.example +++ b/packages/code/.env.example @@ -81,6 +81,12 @@ JIRA_API_TOKEN=your-api-token-here # Which AI agent to use: claude-code | opencode | codex | cursor | grok | deepseek # (also: gemini | kimi | qwen | goose | kilo-code | cline | pi) # Defaults to 'claude-code' if not specified +# +# Accepts a comma-separated, priority-ordered list for automatic failover in +# worker mode: when the first harness hits its usage limit, the worker +# switches to the next entry and fails back to the first once its window +# resets. Unknown or not-installed entries are warned about and skipped. +# AGENT_HARNESS=claude-code,codex AGENT_HARNESS=claude-code # Optional: Path to the agent CLI executable. diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 6b1d471..2136da9 100755 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -1432,6 +1432,7 @@ async function processSingleTask(taskKey: string, taskIndex = 0, totalTasks = 1) origin: scheduledAutomationId ? "scheduled" : "task", taskKey: workflowKey, tracker: process.env.TASK_TRACKER || "jira", + harness: resolvedAgent.harness.name, ...(scheduledAutomationId ? { automationId: scheduledAutomationId } : {}), }); diff --git a/packages/code/src/lib/address-review.ts b/packages/code/src/lib/address-review.ts index e7972c8..2d9ed5c 100644 --- a/packages/code/src/lib/address-review.ts +++ b/packages/code/src/lib/address-review.ts @@ -485,6 +485,7 @@ export async function addressReview( repo: `${owner}/${repo}`, prNumber, branch: pr.head.ref, + harness: resolveHarness({ warnDeprecated: false }).harness.name, }); recordRunStage("change_request", { status: "succeeded", diff --git a/packages/code/src/lib/harness-failover.ts b/packages/code/src/lib/harness-failover.ts new file mode 100644 index 0000000..597503b --- /dev/null +++ b/packages/code/src/lib/harness-failover.ts @@ -0,0 +1,277 @@ +/** + * Harness failover state machine for worker mode. + * + * Given a priority-ordered harness chain (from `AGENT_HARNESS=claude-code,codex`), + * tracks the active harness and per-harness usage-limit windows so the queue + * keeps processing on a fallback when the primary hits its limit, and returns + * to the primary once its window elapses (failback). + * + * This module is the in-memory source of truth. Persistence hooks let the + * webhook server mirror the state into the queue database (`webhook_meta` + * keys, per-harness) so a worker restart recovers the windows and the active + * harness instead of immediately retrying a still-limited agent. + * + * Selection is deterministic: the active harness is always the highest-priority + * (lowest-index) entry whose limit window has elapsed, so failback to the + * primary happens automatically as soon as its window ends. + */ + +import type { HarnessChainEntry } from "@devintern/agent-harness"; + +/** What happened after a harness reported a usage limit. */ +export type FailoverOutcome = + | { + kind: "switched"; + from: string; + to: string; + /** Epoch ms when the from-harness window ends. */ + untilMs: number; + } + | { + kind: "stayed"; + entry: string; + untilMs: number; + } + | { + kind: "exhausted"; + /** Every chain entry is limited; the caller must pause the queue. */ + untilMs: number; + }; + +/** Persistence + clock hooks injected by the host (webhook server / tests). */ +export interface HarnessFailoverOptions { + /** Priority-ordered usable chain entries (from `resolveHarnessChain`). */ + entries: HarnessChainEntry[]; + /** Injectable clock (epoch ms). Defaults to `Date.now`. */ + now?: () => number; + /** Persist a per-harness limit window (epoch ms). */ + persistLimit?: (harness: string, untilMs: number) => void; + /** Remove a harness's persisted limit window. */ + clearPersistedLimit?: (harness: string) => void; + /** Persist the current active harness name. */ + persistActive?: (harness: string) => void; + /** Log line override (tests). Defaults to `console.log`. */ + log?: (message: string) => void; +} + +/** + * Manage the active harness of a chain and its per-harness limit windows. + */ +export class HarnessFailover { + private readonly entries: HarnessChainEntry[]; + private readonly limited = new Map(); + private activeIndex = 0; + private readonly now: () => number; + private readonly persistLimit?: (harness: string, untilMs: number) => void; + private readonly clearPersistedLimit?: (harness: string) => void; + private readonly persistActive?: (harness: string) => void; + private readonly log: (message: string) => void; + + constructor(options: HarnessFailoverOptions) { + if (options.entries.length === 0) { + throw new Error("HarnessFailover requires at least one chain entry"); + } + this.entries = options.entries; + this.now = options.now ?? Date.now; + this.persistLimit = options.persistLimit; + this.clearPersistedLimit = options.clearPersistedLimit; + this.persistActive = options.persistActive; + this.log = options.log ?? ((message: string) => console.log(message)); + } + + /** The currently active chain entry. */ + get active(): HarnessChainEntry { + return this.entries[this.activeIndex]!; + } + + /** Canonical name of the active harness. */ + get activeName(): string { + return this.active.name; + } + + /** Whether the active harness is the first (priority) entry. */ + get onPrimary(): boolean { + return this.activeIndex === 0; + } + + /** Ordered canonical names, for logging ("a → b → c"). */ + describeChain(): string { + return this.entries.map((e) => e.name).join(" → "); + } + + /** + * Snapshot of the tracked per-harness limit windows (epoch ms). + * + * Entries whose window has already ended are left in place until + * {@link windowElapsed} clears them, so a host timer can observe the + * expiration and run the failback. + */ + windows(): Record { + return Object.fromEntries(this.limited); + } + + /** + * Seed state from persisted per-harness windows (queue DB) after a restart. + * + * Future windows for harnesses still in the chain are restored; expired + * windows and windows for harnesses no longer in the chain (list reordered + * or shortened) are dropped from persistence as stale state. Selection + * afterwards is priority-driven: the highest-priority entry without an + * open window becomes active, so a still-limited primary is never retried + * and a fallback from before the restart is kept when it is the best + * available entry. The persisted active name is only used to warn when it + * has left the chain. + * + * @param windows - Persisted harness → limit-until (epoch ms) map + * @param activeName - Persisted active harness name, when known + * @returns Warning lines for stale state the caller should log. + */ + restore(windows: Record, activeName?: string | null): string[] { + const warnings: string[] = []; + const nowMs = this.now(); + + for (const [harness, untilMs] of Object.entries(windows)) { + if (!this.entries.some((e) => e.name === harness)) { + warnings.push( + `Failover state references harness "${harness}", which is not in the current AGENT_HARNESS chain; ignoring its limit window.`, + ); + this.clearPersistedLimit?.(harness); + continue; + } + if (untilMs > nowMs) { + this.limited.set(harness, untilMs); + } else { + // Stale window (already elapsed while the worker was down) — clear it. + this.clearPersistedLimit?.(harness); + } + } + + if (activeName && !this.entries.some((e) => e.name === activeName)) { + warnings.push( + `Persisted active harness "${activeName}" is not in the current AGENT_HARNESS chain; falling back to the highest-priority available entry.`, + ); + } + + const target = this.selectAvailable(); + if (target) { + this.activeIndex = this.entries.indexOf(target); + } else { + // Everything limited: park on the primary; the queue starts paused and + // the resume path reselects when the earliest window elapses. + this.activeIndex = 0; + } + return warnings; + } + + /** Whether the named harness currently has an open limit window. */ + isLimited(harness: string): boolean { + const until = this.limited.get(harness); + return until !== undefined && until > this.now(); + } + + /** + * Record a usage limit for the active harness and fail over when possible. + * + * Extends an existing window (keeps the furthest reset), then switches to + * the highest-priority entry that is not limited. When every entry is + * limited the outcome is `exhausted` and the caller must pause its queue + * until {@link earliestResetMs}. + * + * @param resetUntilMs - Epoch ms when the active harness's window ends + * (already resolved from the reset hint or a fallback cooldown). + * @returns What the failover decided. + */ + reportUsageLimit(resetUntilMs: number): FailoverOutcome { + const from = this.active; + this.setWindow(from.name, resetUntilMs); + + const target = this.selectAvailable(); + if (!target) { + const untilMs = this.earliestResetMs() ?? resetUntilMs; + this.log( + `⛔ All harnesses in the chain are usage-limited (${this.describeChain()}); ` + + `resuming when the earliest window ends at ${new Date(untilMs).toISOString()}.`, + ); + return { kind: "exhausted", untilMs }; + } + + const toIndex = this.entries.indexOf(target); + this.activeIndex = toIndex; + this.persistActive?.(this.activeName); + + if (target.name !== from.name) { + const fallbackPosition = + toIndex === 0 ? "primary" : `fallback ${toIndex + 1}/${this.entries.length}`; + this.log( + `🔁 ${from.name} hit a usage limit (until ${new Date(resetUntilMs).toISOString()}) — ` + + `failing over to ${target.name} (${fallbackPosition} in the AGENT_HARNESS chain).`, + ); + return { kind: "switched", from: from.name, to: target.name, untilMs: resetUntilMs }; + } + + return { kind: "stayed", entry: target.name, untilMs: resetUntilMs }; + } + + /** + * Clear an elapsed limit window and fail back when that unlocks a + * higher-priority harness. + * + * Called by the host timer when a persisted window ends. If the primary + * (priority) harness becomes available again while a fallback is active, + * the active harness returns to it — that is the failback. + * + * @param harness - Harness whose window elapsed. + * @returns The harness now active, if a switch happened; otherwise null. + */ + windowElapsed(harness: string): string | null { + const hadWindow = this.limited.delete(harness); + if (hadWindow) { + this.clearPersistedLimit?.(harness); + } + + const target = this.selectAvailable(); + if (!target || target.name === this.activeName) { + return null; + } + + const previous = this.active; + this.activeIndex = this.entries.indexOf(target); + this.persistActive?.(this.activeName); + this.log( + target.name === this.entries[0]!.name + ? `⏪ ${harness} usage-limit window elapsed — failing back to primary harness ${target.name} (from ${previous.name}).` + : `🔁 ${harness} usage-limit window elapsed — resuming on ${target.name} (from ${previous.name}).`, + ); + return target.name; + } + + /** + * Epoch ms when the earliest open window ends (null when none open). + * + * Always in the future when non-null; expired-but-uncleaned windows are + * ignored. The host arms its resume/failback timer at this instant. + */ + earliestResetMs(): number | null { + const nowMs = this.now(); + const open = [...this.limited.values()].filter((until) => until > nowMs); + return open.length > 0 ? Math.min(...open) : null; + } + + /** Whether every chain entry currently has an open limit window. */ + allLimited(): boolean { + return this.entries.every((e) => this.isLimited(e.name)); + } + + /** Highest-priority entry without an open window, or null when all limited. */ + private selectAvailable(): HarnessChainEntry | null { + return this.entries.find((e) => !this.isLimited(e.name)) ?? null; + } + + /** Store a window, keeping the furthest reset when one is already open. */ + private setWindow(harness: string, untilMs: number): void { + const existing = this.limited.get(harness) ?? 0; + const until = Math.max(existing, untilMs); + this.limited.set(harness, until); + this.persistLimit?.(harness, until); + } +} diff --git a/packages/code/src/lib/review-polling-acquirer.ts b/packages/code/src/lib/review-polling-acquirer.ts index 2f89689..3ef439c 100644 --- a/packages/code/src/lib/review-polling-acquirer.ts +++ b/packages/code/src/lib/review-polling-acquirer.ts @@ -25,6 +25,8 @@ import { spawn } from "child_process"; +import { parseHarnessList } from "@devintern/agent-harness"; + import { parseEnvInteger } from "./env-integer"; import type { RunStore } from "./run-recorder"; import type { WebhookQueue } from "./webhook-queue"; @@ -631,7 +633,7 @@ export class ReviewPollingAcquirer implements Acquirer { repo, prNumber, branch: fresh.head.ref, - harness: this.options.harness ?? process.env.AGENT_HARNESS ?? "claude-code", + harness: this.options.harness ?? parseHarnessList(process.env.AGENT_HARNESS)[0], attempt, }) ?? null; } catch (error) { diff --git a/packages/code/src/lib/webhook-queue.ts b/packages/code/src/lib/webhook-queue.ts index 739591d..d855af8 100644 --- a/packages/code/src/lib/webhook-queue.ts +++ b/packages/code/src/lib/webhook-queue.ts @@ -315,6 +315,57 @@ export class WebhookQueue { return Number.isFinite(ms) ? ms : null; } + /** Every persisted per-harness rate-limit window, keyed by harness name. */ + getAllRateLimits(): Record { + const rows = this.db + .query(`SELECT key, value FROM webhook_meta WHERE key LIKE 'rate_limit:%'`) + .all() as { key: string; value: string }[]; + const result: Record = {}; + for (const row of rows) { + const ms = Number(row.value); + if (Number.isFinite(ms)) { + result[row.key.slice("rate_limit:".length)] = ms; + } + } + return result; + } + + /** Meta key for the failover chain's active harness. */ + private activeHarnessKey(): string { + return "failover:active_harness"; + } + + /** + * Persist the failover chain's active harness so a worker restart resumes + * on it instead of snapping back to the primary mid-window. + * + * @param harness - Canonical harness name (e.g. `codex`) + */ + setActiveHarness(harness: string): void { + this.db.run( + `INSERT INTO webhook_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + [this.activeHarnessKey(), harness], + ); + } + + /** Forget the persisted active harness (e.g. when the chain changed). */ + clearActiveHarness(): void { + this.db.run(`DELETE FROM webhook_meta WHERE key = ?`, [this.activeHarnessKey()]); + } + + /** + * Read the persisted active harness of the failover chain. + * + * @returns Canonical harness name, or `null` when none was persisted. + */ + getActiveHarness(): string | null { + const row = this.db + .query(`SELECT value FROM webhook_meta WHERE key = ?`) + .get(this.activeHarnessKey()) as { value: string } | undefined; + return row?.value ?? null; + } + /** * Check whether a provider-issued event id was already handled. * diff --git a/packages/code/src/webhook-server.ts b/packages/code/src/webhook-server.ts index ec738c7..649d8cd 100644 --- a/packages/code/src/webhook-server.ts +++ b/packages/code/src/webhook-server.ts @@ -18,11 +18,13 @@ import { detectUsageLimit, resetHintToMs, resolveHarness, + resolveHarnessChain, spawnAgent, reapTree, resolveExecutablePathWithRetry, UsageLimitError, } from "@devintern/agent-harness"; +import type { ResolvedHarness } from "@devintern/agent-harness"; import { buildHeadlessAgentArgs, HEADLESS_AGENT_STDIO } from "./lib/agent-spawn"; import { parseEnvInteger } from "./lib/env-integer"; import { resolveAgentModel } from "./lib/agent-model"; @@ -30,6 +32,7 @@ import { getSandbox } from "./lib/sandbox"; import { GitHubAppAuth } from "./lib/github-app-auth"; import { GitHubReviewsClient } from "./lib/github-reviews"; import { LEGACY_DB_PATH, WebhookQueue, resolveQueueDbPath } from "./lib/webhook-queue"; +import { HarnessFailover } from "./lib/harness-failover"; import { formatReviewPrompt } from "./lib/review-formatter"; import { Utils } from "./lib/utils"; import { isCommitAlreadyComplete, runAgentHarnessToFixGitHook } from "./lib/git-hook-fixer"; @@ -76,68 +79,121 @@ let webhookQueue: WebhookQueue | null = null; // Fallback cooldown when a usage-limit reset hint can't be parsed. const RATE_LIMIT_FALLBACK_MS = 60 * 60 * 1000; // 1 hour -// In-memory mirror of the active rate-limit window for the current harness. -let rateLimitedUntil: number | null = null; +// Failover chain state. The HarnessFailover instance is the in-memory source +// of truth; per-harness limit windows and the active harness are mirrored +// into the queue DB (keyed by harness) so they survive a restart. Assigned in +// `startWebhookServer` (lazily via `ensureFailover` for direct module use). +let failover: HarnessFailover | null = null; let rateLimitResumeTimer: ReturnType | null = null; // Cleanup rate limiter periodically setInterval(() => rateLimiter.cleanup(), 60000); // Note: We use a single reusable worktree, so no periodic cleanup needed -/** Name of the agent harness this server drives (e.g. `claude-code`). */ +/** + * Lazily build the failover state from the `AGENT_HARNESS` chain. + * + * Startup always initializes with installability checks; this fallback covers + * direct module use before `startWebhookServer` runs (e.g. in tests). + */ +function ensureFailover(): HarnessFailover { + if (failover) { + return failover; + } + const chain = resolveHarnessChain({ checkInstalled: false }); + failover = new HarnessFailover({ entries: chain.entries }); + return failover; +} + +/** Name of the agent harness currently driving this server (e.g. `claude-code`). */ function currentHarnessName(): string { - return resolveHarness().harness.name; + return ensureFailover().activeName; } /** - * Pause the review queue until the current harness's usage limit resets. + * Resolve the active harness and its executable path for an agent spawn. * - * Idempotent: extends the window if a later reset arrives. The persisted state - * is keyed by harness so a restart with a different `AGENT_HARNESS` is not - * wrongly blocked. + * Per-harness env overrides (`_CLI_PATH`) were already applied when + * the chain was resolved, so every spawn uses the right CLI for whichever + * harness failover selected. `AGENT_MODEL` is read at spawn time and applies + * to the active harness (the string is harness-specific by nature). + * + * @returns The resolved harness and executable path to spawn. + */ +function resolveActiveHarness(): ResolvedHarness { + const active = ensureFailover().active; + return { harness: active.harness, path: active.path }; +} + +/** + * Handle a usage-limit report from the active harness. + * + * With a multi-harness `AGENT_HARNESS` chain, fail over to the highest-priority + * harness whose limit window has elapsed and keep processing — the queue only + * pauses when every harness in the chain is limited (which is also the exact + * behavior of a single-harness configuration). The window is persisted per + * harness and the failback timer armed, so the worker returns to the primary + * harness as soon as its window ends. * * @param resetHint - Human-readable reset hint from the agent output */ -function enterRateLimitPause(resetHint?: string): void { - const harness = currentHarnessName(); +function handleUsageLimit(resetHint?: string): void { + const manager = ensureFailover(); + const harness = manager.activeName; const until = resetHintToMs(resetHint, Date.now()) ?? Date.now() + RATE_LIMIT_FALLBACK_MS; - // Keep the latest (furthest) reset if one is already active. - rateLimitedUntil = Math.max(rateLimitedUntil ?? 0, until); - webhookQueue?.setRateLimit(harness, rateLimitedUntil); - - if (!reviewQueue.isPaused) { - reviewQueue.pause(); + const outcome = manager.reportUsageLimit(until); + if (outcome.kind === "exhausted") { + if (!reviewQueue.isPaused) { + reviewQueue.pause(); + } + const waitMs = Math.max(0, outcome.untilMs - Date.now()); + console.warn( + `⏳ ${harness} hit a usage limit${resetHint ? ` (resets ${resetHint})` : ""} and no fallback harness is available. ` + + `Pausing webhook queue until ${new Date(outcome.untilMs).toISOString()} (~${Math.round(waitMs / 60000)} min). ` + + `Queued and incoming events will wait and drain on resume.`, + ); } - - const waitMs = Math.max(0, rateLimitedUntil - Date.now()); - const resetAtIso = new Date(rateLimitedUntil).toISOString(); - console.warn( - `⏳ ${harness} hit a usage limit${resetHint ? ` (resets ${resetHint})` : ""}. ` + - `Pausing webhook queue until ${resetAtIso} (~${Math.round(waitMs / 60000)} min). ` + - `Queued and incoming events will wait and drain on resume.`, - ); - - scheduleRateLimitResume(); + armFailoverTimers(); } -/** (Re)arm the timer that resumes the queue when the rate-limit window ends. */ -function scheduleRateLimitResume(): void { +/** + * Arm the failback/resume timer at the earliest open limit window. + * + * When it fires, each elapsed window is cleared (persisted too) — a failback + * to the primary harness is logged when that unlocks it — and a queue paused + * for all-limited exhaustion resumes. + */ +function armFailoverTimers(): void { if (rateLimitResumeTimer) { clearTimeout(rateLimitResumeTimer); + rateLimitResumeTimer = null; } - if (rateLimitedUntil === null) { + const manager = failover; + const nextMs = manager?.earliestResetMs() ?? null; + if (!manager || nextMs === null) { return; } - const waitMs = Math.max(0, rateLimitedUntil - Date.now()); - rateLimitResumeTimer = setTimeout(() => { - const harness = currentHarnessName(); - rateLimitedUntil = null; - rateLimitResumeTimer = null; - webhookQueue?.clearRateLimit(harness); - console.log(`▶️ Usage limit window elapsed for ${harness} — resuming webhook queue`); - reviewQueue.start(); - }, waitMs); + rateLimitResumeTimer = setTimeout( + () => { + rateLimitResumeTimer = null; + if (!failover) { + return; + } + const nowMs = Date.now(); + for (const [harness, until] of Object.entries(failover.windows())) { + if (until <= nowMs) { + failover.windowElapsed(harness); + } + } + if (reviewQueue.isPaused && !failover.allLimited()) { + console.log(`▶️ Usage-limit windows elapsed — resuming webhook queue`); + reviewQueue.start(); + } + armFailoverTimers(); + }, + Math.max(0, nextMs - Date.now()), + ); } /** @@ -489,9 +545,10 @@ async function processReviewWithPersistence( } } catch (error) { if (error instanceof UsageLimitError) { - // Deferred by an account-global usage limit — pause and re-queue for - // after reset instead of counting a failure. - enterRateLimitPause(error.resetHint); + // Deferred by an account-global usage limit — fail over to the next + // harness (or pause and re-queue for after reset) instead of counting + // a failure. + handleUsageLimit(error.resetHint); if (eventId && webhookQueue) { webhookQueue.requeuePending(eventId); } @@ -532,7 +589,7 @@ async function processIssueCommentWithPersistence( } } catch (error) { if (error instanceof UsageLimitError) { - enterRateLimitPause(error.resetHint); + handleUsageLimit(error.resetHint); if (eventId && webhookQueue) { webhookQueue.requeuePending(eventId); } @@ -802,7 +859,7 @@ async function processReviewAsync( const autoReviewOutputDir = `/tmp/devintern-auto-review-${prNumber}`; const baseBranch = event.pull_request.base.ref; - const { harness: reviewHarness, path: reviewPath } = resolveHarness(); + const { harness: reviewHarness, path: reviewPath } = resolveActiveHarness(); try { const autoReviewResult = await runAutoReviewLoop({ repository: `${owner}/${repo}`, @@ -858,7 +915,8 @@ async function processReviewAsync( const hitMaxTurns = agentResult.maxTurnsReached === true; // A usage limit is account-global: don't burn this event as a failure — - // signal the wrapper to pause the queue and re-queue it for after reset. + // signal the wrapper to fail over to the next harness (or pause the queue + // and re-queue the event for after reset). if (agentResult.usageLimited) { throw new UsageLimitError(agentResult.usageResetHint); } @@ -874,7 +932,7 @@ async function processReviewAsync( // Get hook retries configuration const hookRetries = parseInt(process.env.HOOK_RETRIES || "10", 10); - const { harness, path: executablePath } = resolveHarness(); + const { harness, path: executablePath } = resolveActiveHarness(); const maxTurns = parseInt(process.env.CLAUDE_MAX_TURNS || "500", 10); // Verify Agent didn't switch branches during execution (e.g., checking out main for comparison) @@ -1077,7 +1135,7 @@ async function processReviewAsync( console.log("\n🔄 Running auto-review loop (without pushing)..."); const autoReviewOutputDir = `/tmp/devintern-auto-review-${prNumber}`; const baseBranchForReview = event.pull_request.base.ref; - const { harness: reviewHarness2, path: reviewPath2 } = resolveHarness(); + const { harness: reviewHarness2, path: reviewPath2 } = resolveActiveHarness(); try { const autoReviewResult = await runAutoReviewLoop({ repository: `${owner}/${repo}`, @@ -1188,6 +1246,12 @@ async function processReviewAsync( console.log(`\n✅ Successfully addressed review for PR #${prNumber}`); } catch (error) { + if (error instanceof UsageLimitError) { + // Account-global usage limit: propagate to the persistence wrapper so + // it can fail over to the next harness (or pause + re-queue until the + // window resets) without burning the event as a failure. + throw error; + } console.error(`❌ Error processing review: ${(error as Error).message}`); if (config.debug) { console.error((error as Error).stack); @@ -1244,7 +1308,7 @@ async function runAgentHarnessForReview( usageLimited?: boolean; usageResetHint?: string; }> { - const { harness, path: executablePath } = resolveHarness(); + const { harness, path: executablePath } = resolveActiveHarness(); // Wait out any in-progress CLI auto-update swap before spawning, so a // transient `spawn ENOENT` doesn't abort the review. const resolvedPath = await resolveExecutablePathWithRetry(executablePath, { @@ -1400,18 +1464,24 @@ async function runAgentHarnessForReview( }); } -/** Return JSON health payload including webhook queue stats. */ +/** Return JSON health payload including webhook queue stats and failover state. */ function handleHealthCheck(): Response { const queueStats = webhookQueue?.getStats() || { pending: 0, processing: 0, failed: 0, }; + const manager = failover; return jsonResponse({ status: "ok", timestamp: new Date().toISOString(), version: "1.0.0", queue: queueStats, + harness: { + active: manager?.activeName ?? currentHarnessName(), + chain: manager?.describeChain() ?? currentHarnessName(), + rateLimitedUntil: manager?.windows() ?? {}, + }, }); } @@ -1509,22 +1579,44 @@ export async function startWebhookServer( ); } - // Re-apply a usage-limit pause if the current harness is still rate-limited - // from before a restart. Keyed by harness so switching AGENT_HARNESS clears it. - const harness = currentHarnessName(); - const persistedLimit = webhookQueue.getRateLimit(harness); - if (persistedLimit && persistedLimit > Date.now()) { - rateLimitedUntil = persistedLimit; + // Resolve the AGENT_HARNESS failover chain. Unknown or not-installed + // entries warn and are skipped rather than failing startup; a single value + // yields exactly one entry (backward compatible). + const chain = resolveHarnessChain(); + for (const issue of chain.issues) { + console.warn(`⚠️ ${issue.message}`); + } + failover = new HarnessFailover({ + entries: chain.entries, + persistLimit: (harnessName, untilMs) => webhookQueue?.setRateLimit(harnessName, untilMs), + clearPersistedLimit: (harnessName) => webhookQueue?.clearRateLimit(harnessName), + persistActive: (harnessName) => webhookQueue?.setActiveHarness(harnessName), + }); + console.log( + ` Agent harness: ${failover.describeChain()}${ + chain.multiHarness ? " (failover enabled)" : "" + }`, + ); + + // Recover the failover state persisted before a restart: per-harness limit + // windows and the active harness. The in-memory selection is the runtime + // source of truth — stale state (a harness no longer in the chain, or an + // already-elapsed window) is dropped with a warning. + for (const warning of failover.restore( + webhookQueue.getAllRateLimits(), + webhookQueue.getActiveHarness(), + )) { + console.warn(`⚠️ ${warning}`); + } + if (failover.allLimited()) { + const untilMs = failover.earliestResetMs() ?? Date.now(); reviewQueue.pause(); - scheduleRateLimitResume(); console.warn( - `⏳ ${harness} is still rate-limited until ${new Date(persistedLimit).toISOString()} ` + + `⏳ Every harness in the chain is rate-limited until ${new Date(untilMs).toISOString()} ` + `— webhook queue starts paused; recovered events will wait.`, ); - } else if (persistedLimit) { - // Stale window (already elapsed) — clear it. - webhookQueue.clearRateLimit(harness); } + armFailoverTimers(); // Recover pending/processing events from previous runs const pendingEvents = webhookQueue.getPendingEvents(); diff --git a/packages/code/tests/harness-failover.test.ts b/packages/code/tests/harness-failover.test.ts new file mode 100644 index 0000000..51e8464 --- /dev/null +++ b/packages/code/tests/harness-failover.test.ts @@ -0,0 +1,250 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +import { getHarness } from "@devintern/agent-harness"; +import type { HarnessChainEntry } from "@devintern/agent-harness"; + +import { HarnessFailover } from "../src/lib/harness-failover"; +import { WebhookQueue } from "../src/lib/webhook-queue"; + +/** Build a chain entry from a registered harness name. */ +function entry(name: string): HarnessChainEntry { + const harness = getHarness(name); + if (!harness) { + throw new Error(`Harness "${name}" is not registered`); + } + return { name: harness.name, harness, path: harness.defaultPath, installed: true }; +} + +/** Build a failover over the given chain with captured logs and persistence. */ +function makeFailover(names: string[], now = () => 1000) { + const lines: string[] = []; + const limits: Record = {}; + const cleared: string[] = []; + const state = { active: null as string | null }; + const failover = new HarnessFailover({ + entries: names.map(entry), + now, + persistLimit: (harness, untilMs) => { + limits[harness] = untilMs; + }, + clearPersistedLimit: (harness) => { + cleared.push(harness); + delete limits[harness]; + }, + persistActive: (harness) => { + state.active = harness; + }, + log: (message) => lines.push(message), + }); + return { failover, lines, limits, cleared, state }; +} + +describe("HarnessFailover", () => { + test("single-entry chain is exhausted when the harness is limited", () => { + const h = makeFailover(["claude-code"]); + const outcome = h.failover.reportUsageLimit(2000); + expect(outcome.kind).toBe("exhausted"); + expect(outcome.kind === "exhausted" && outcome.untilMs).toBe(2000); + expect(h.limits["claude-code"]).toBe(2000); + }); + + test("fails over to the next entry when the primary is limited", () => { + const h = makeFailover(["claude-code", "codex"]); + const outcome = h.failover.reportUsageLimit(2000); + expect(outcome).toEqual({ + kind: "switched", + from: "claude-code", + to: "codex", + untilMs: 2000, + }); + expect(h.failover.activeName).toBe("codex"); + expect(h.failover.onPrimary).toBe(false); + expect(h.state.active).toBe("codex"); + expect(h.limits["claude-code"]).toBe(2000); + expect(h.lines.join("\n")).toContain("failing over to codex"); + }); + + test("advances again when the fallback also hits a limit mid-task", () => { + const h = makeFailover(["claude-code", "codex", "grok"]); + h.failover.reportUsageLimit(2000); + const second = h.failover.reportUsageLimit(3000); + expect(second).toEqual({ kind: "switched", from: "codex", to: "grok", untilMs: 3000 }); + expect(h.failover.activeName).toBe("grok"); + expect(h.limits["codex"]).toBe(3000); + }); + + test("exhausts when every entry is limited and reports the earliest reset", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(5000); + const outcome = h.failover.reportUsageLimit(3000); + expect(outcome.kind).toBe("exhausted"); + expect(outcome.kind === "exhausted" && outcome.untilMs).toBe(3000); + expect(h.failover.earliestResetMs()).toBe(3000); + expect(h.failover.allLimited()).toBe(true); + }); + + test("keeps the furthest reset when a window is extended", () => { + const h = makeFailover(["claude-code"]); + h.failover.reportUsageLimit(5000); + h.failover.reportUsageLimit(3000); + expect(h.failover.windows()["claude-code"]).toBe(5000); + expect(h.limits["claude-code"]).toBe(5000); + }); + + test("fails back to the primary when its window elapses", () => { + const h = makeFailover(["claude-code", "codex", "grok"]); + h.failover.reportUsageLimit(5000); // claude limited → codex + h.failover.reportUsageLimit(3000); // codex limited → grok + expect(h.failover.activeName).toBe("grok"); + + let now = 3500; + const switched = h.failover.windowElapsed("codex"); + expect(switched).toBe("codex"); + expect(h.failover.activeName).toBe("codex"); + expect(h.cleared).toContain("codex"); + expect(h.lines.join("\n")).toContain("resuming on codex"); + + now = 6000; + const failback = h.failover.windowElapsed("claude-code"); + expect(failback).toBe("claude-code"); + expect(h.failover.activeName).toBe("claude-code"); + expect(h.failover.onPrimary).toBe(true); + expect(h.lines.join("\n")).toContain("failing back to primary harness claude-code"); + void now; + }); + + test("windowElapsed is a no-op when no higher-priority harness unlocks", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(5000); // → codex, claude limited + expect(h.failover.windowElapsed("grok")).toBeNull(); + expect(h.failover.activeName).toBe("codex"); + }); + + test("persistence hooks receive every mutation", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.reportUsageLimit(2000); + h.failover.windowElapsed("claude-code"); + expect(h.limits["claude-code"]).toBeUndefined(); + expect(h.cleared).toContain("claude-code"); + expect(h.state.active).toBe("claude-code"); + }); + + test("restore seeds windows and honors a persisted active harness", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ "claude-code": 5000 }, "codex"); + expect(warnings).toEqual([]); + expect(h.failover.activeName).toBe("codex"); + expect(h.failover.isLimited("claude-code")).toBe(true); + expect(h.failover.allLimited()).toBe(false); + }); + + test("restore warns and falls back when the persisted active left the chain", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({}, "grok"); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("grok"); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore ignores windows for harnesses outside the chain", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ grok: 5000 }, null); + expect(warnings).toHaveLength(1); + expect(h.failover.isLimited("grok")).toBe(false); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore drops already-elapsed windows", () => { + const h = makeFailover(["claude-code", "codex"]); + const warnings = h.failover.restore({ "claude-code": 500 }, null); + expect(warnings).toEqual([]); + expect(h.failover.isLimited("claude-code")).toBe(false); + expect(h.cleared).toContain("claude-code"); + expect(h.failover.activeName).toBe("claude-code"); + }); + + test("restore parks on the primary when every entry is limited", () => { + const h = makeFailover(["claude-code", "codex"]); + h.failover.restore({ "claude-code": 5000, codex: 4000 }, null); + expect(h.failover.allLimited()).toBe(true); + expect(h.failover.earliestResetMs()).toBe(4000); + }); + + test("describeChain renders the priority order", () => { + const h = makeFailover(["claude-code", "codex"]); + expect(h.failover.describeChain()).toBe("claude-code → codex"); + }); +}); + +describe("WebhookQueue failover persistence", () => { + let dbPath: string; + + beforeEach(() => { + dbPath = join(tmpdir(), `hf-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`); + }); + + afterEach(() => { + for (const suffix of ["", "-wal", "-shm"]) { + const file = `${dbPath}${suffix}`; + if (existsSync(file)) { + rmSync(file, { force: true }); + } + } + }); + + test("active harness and windows survive a reopen", () => { + const queue = new WebhookQueue({ dbPath }); + queue.setRateLimit("claude-code", 1234); + queue.setRateLimit("codex", 5678); + queue.setActiveHarness("codex"); + + const reopened = new WebhookQueue({ dbPath }); + expect(reopened.getAllRateLimits()).toEqual({ "claude-code": 1234, codex: 5678 }); + expect(reopened.getActiveHarness()).toBe("codex"); + + reopened.clearRateLimit("claude-code"); + expect(reopened.getAllRateLimits()).toEqual({ codex: 5678 }); + + reopened.clearActiveHarness(); + expect(reopened.getActiveHarness()).toBeNull(); + reopened.close(); + }); + + test("getAllRateLimits is empty when nothing is persisted", () => { + const queue = new WebhookQueue({ dbPath }); + expect(queue.getAllRateLimits()).toEqual({}); + expect(queue.getActiveHarness()).toBeNull(); + queue.close(); + }); + + test("a failover round-trip through the queue restores correctly", () => { + const first = new WebhookQueue({ dbPath }); + const h = makeFailover(["claude-code", "codex"]); + // Simulate the webhook server wiring failover persistence to the queue. + const manager = new HarnessFailover({ + entries: ["claude-code", "codex"].map(entry), + now: () => 1000, + persistLimit: (harness, untilMs) => first.setRateLimit(harness, untilMs), + clearPersistedLimit: (harness) => first.clearRateLimit(harness), + persistActive: (harness) => first.setActiveHarness(harness), + log: () => {}, + }); + void h; + manager.reportUsageLimit(9000); + first.close(); + + const second = new WebhookQueue({ dbPath }); + const restored = new HarnessFailover({ + entries: ["claude-code", "codex"].map(entry), + now: () => 2000, + log: () => {}, + }); + restored.restore(second.getAllRateLimits(), second.getActiveHarness()); + expect(restored.activeName).toBe("codex"); + expect(restored.isLimited("claude-code")).toBe(true); + second.close(); + }); +});