[DEV-106] Automatic failover across multiple agent harnesses when usage limits are hit

docs/code/configuration.md

@@ -365,6 +365,30 @@ Common `AGENT_HARNESS` values include `claude-code`, `opencode`, `codex`, `curso
 
 **Qwen note:** Qwen Code accepts a model via its `--model` flag (e.g. `qwen3-coder-plus`) — set it with `AGENT_MODEL`; you can also keep the model in `~/.qwen/settings.json`.
 
+### Failover across multiple harnesses
+
+`AGENT_HARNESS` accepts a comma-separated, priority-ordered list so the unattended worker keeps processing when an agent hits its usage limit:
+
+```bash
+# .devintern-code/.env
+AGENT_HARNESS=claude-code,codex
+```
+
+The first entry is your preferred harness; later entries are fallbacks in priority order. A single value behaves exactly as before.
+
+**Failover behavior (worker mode):**
+
+- At startup every entry is checked against the harness registry and your machine: unknown or not-installed entries produce a clear warning and are skipped, and the effective chain is logged (e.g. `Agent harness: claude-code → codex (failover enabled)`).
+- When the active harness reports a usage/rate limit, the worker records its reset window (parsed from the limit output; a 1-hour cooldown applies when no timer is parseable, e.g. monthly spend limits) and switches to the highest-priority harness that still has capacity. Queued and incoming events continue processing on the fallback without losing anything.
+- When the primary harness's window elapses, the worker automatically fails back to it and logs the switch. Fallback agents hitting their own limits mid-queue advance the chain again.
+- If every harness in the chain is limited at once, the queue pauses exactly as it does for a single harness and resumes when the earliest window ends.
+- Failover state (active harness + per-harness windows) persists in the queue database, so restarting the worker resumes on the right harness instead of retrying a still-limited agent.
+- Which harness executed each run is recorded in run records, and `/health` on the webhook server reports the active harness, the chain, and open limit windows.
+
+One-shot runs (`devintern TASK-123`), review helpers, and other non-worker flows always use the first (priority) entry; multi-harness failover there is out of scope.
+
+**Per-harness overrides inside a list:** `<HARNESS>_CLI_PATH` (e.g. `CODEX_CLI_PATH`) resolves per active harness at spawn time. The global `AGENT_CLI_PATH` applies to the first entry only, so a stale global override cannot leak onto a fallback agent. `AGENT_MODEL` applies to whichever harness is active (the string is harness-specific).
+
 ### Model selection
 
 Set the model the agent harness runs with using `AGENT_MODEL` in `.devintern-code/.env`:

packages/agent-harness/src/harness-chain.ts

@@ -0,0 +1,242 @@
+/**
+ * Priority-ordered harness chain (`AGENT_HARNESS=claude-code,codex`).
+ *
+ * A comma-separated `AGENT_HARNESS` value names a failover chain: the first
+ * entry is the preferred harness, later entries are fallbacks used when an
+ * earlier one hits its usage/rate limit. Parsing and resolution live here so
+ * every consumer (worker failover, readiness probe, one-shot runs) sees the
+ * same list semantics:
+ *
+ *   - Entries are split on commas and trimmed; empty entries are dropped.
+ *   - Aliases (e.g. `agy`, deprecated `gemini`) resolve to canonical names.
+ *   - Duplicate canonical names collapse to the first occurrence, preserving
+ *     the requested priority.
+ *   - An empty/unset value defaults to `["claude-code"]`.
+ *
+ * Resolution validates each name against the registry, resolves its CLI path
+ * per harness (so `AGENT_CLI_PATH` cannot leak across harnesses — harness
+ * specific `<HARNESS>_CLI_PATH` overrides and defaults apply instead), and
+ * checks installability. Unknown or not-installed entries are reported as
+ * issues for the caller to warn about and skipped rather than fatal, unless
+ * that would leave an empty chain (then the full list is kept and the spawn
+ * itself surfaces the real error, matching single-harness behavior).
+ */
+
+import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js";
+import { getHarnessCliCommand, isHarnessCliAvailable } from "./resolver.js";
+import type { AgentHarness } from "./types.js";
+
+/**
+ * Parse a raw (possibly comma-separated) harness list into canonical names.
+ *
+ * Applies registry aliases, drops empty entries, and de-duplicates canonical
+ * names keeping the first occurrence. Returns `[DEFAULT_HARNESS_NAME]` when
+ * nothing usable remains.
+ *
+ * @param raw - Raw `AGENT_HARNESS` value (already env-resolved), may be undefined.
+ * @returns Ordered canonical harness names (priority first).
+ */
+export function parseHarnessList(raw: string | undefined): string[] {
+  const names: string[] = [];
+  const seen = new Set<string>();
+  for (const part of (raw ?? "").split(",")) {
+    const requested = part.trim();
+    if (!requested) {
+      continue;
+    }
+    const canonical = getHarness(requested)?.name ?? requested;
+    if (seen.has(canonical)) {
+      continue;
+    }
+    seen.add(canonical);
+    names.push(canonical);
+  }
+  return names.length > 0 ? names : [DEFAULT_HARNESS_NAME];
+}
+
+/** One ordered, resolved entry of the harness chain. */
+export interface HarnessChainEntry {
+  /** Canonical registry name (after alias resolution). */
+  readonly name: string;
+  readonly harness: AgentHarness;
+  /** Resolved CLI command/path for this harness. */
+  readonly path: string;
+  /** Whether the resolved CLI is installed/reachable on this machine. */
+  readonly installed: boolean;
+}
+
+/** A chain entry that could not be used, with a warning message. */
+export interface HarnessChainIssue {
+  /** Name as written in the list (after alias resolution). */
+  readonly requested: string;
+  readonly reason: "unknown" | "not-installed";
+  readonly message: string;
+}
+
+/** Result of resolving a harness chain. */
+export interface ResolvedHarnessChain {
+  /** Ordered usable entries (priority first). */
+  readonly entries: HarnessChainEntry[];
+  /** Entries dropped from the chain, with warning messages. */
+  readonly issues: HarnessChainIssue[];
+  /** All canonical names as parsed from the raw value (before dropping). */
+  readonly parsed: string[];
+  /** True when the raw value listed more than one harness. */
+  readonly multiHarness: boolean;
+}
+
+/** Options for {@link resolveHarnessChain}. */
+export interface HarnessChainOptions {
+  /** Raw list value; defaults to `AGENT_HARNESS` env (or the default harness). */
+  raw?: string;
+  /**
+   * Probe installability and drop not-installed entries (default true).
+   * When false, every known entry is kept with `installed` reported as-is.
+   */
+  checkInstalled?: boolean;
+  /**
+   * When false, suppress deprecation warnings for aliased harness names
+   * (e.g. `gemini` → `antigravity`). Defaults to true.
+   */
+  warnDeprecated?: boolean;
+  /** Installability predicate override (tests). Defaults to PATH probing. */
+  isInstalled?: (entry: { name: string; path: string }) => boolean;
+}
+
+/**
+ * Resolve the CLI path for one parsed chain entry.
+ *
+ * Harness-specific `<HARNESS>_CLI_PATH` overrides and the harness default
+ * apply; the global `AGENT_CLI_PATH` only applies to the first (priority)
+ * entry so a single-value configuration keeps resolving exactly like
+ * {@link resolveHarness} does today, while a stale global override cannot
+ * stick to a fallback harness selected during failover.
+ *
+ * @param harness - Registered harness for the entry.
+ * @param isPrimary - Whether this is the first (priority) chain entry.
+ * @param warnDeprecated - Whether legacy path env vars may warn.
+ * @returns The CLI command or path to spawn/probe for this entry.
+ */
+function resolveEntryPath(
+  harness: AgentHarness,
+  isPrimary: boolean,
+  warnDeprecated?: boolean,
+): string {
+  if (isPrimary && process.env.AGENT_CLI_PATH) {
+    return process.env.AGENT_CLI_PATH;
+  }
+  return getHarnessCliCommand(harness, { warnDeprecated });
+}
+
+/**
+ * Resolve the priority-ordered harness chain from a comma-separated value.
+ *
+ * Unknown names and (when `checkInstalled`) not-installed CLIs are reported in
+ * `issues` and skipped; if nothing survives, the full parsed list is kept so
+ * spawning fails with the familiar actionable error instead of an empty chain.
+ *
+ * @param options - Raw value override, installability toggles, and test hooks.
+ * @returns The resolved chain with usable entries and dropped-entry issues.
+ */
+export function resolveHarnessChain(options: HarnessChainOptions = {}): ResolvedHarnessChain {
+  const raw = options.raw ?? process.env.AGENT_HARNESS;
+  const parsed = parseHarnessList(raw);
+  const checkInstalled = options.checkInstalled ?? true;
+  const warnDeprecated = options.warnDeprecated;
+
+  const entries: HarnessChainEntry[] = [];
+  const issues: HarnessChainIssue[] = [];
+
+  const buildEntry = (
+    canonical: string,
+    isPrimary: boolean,
+    reportIssues: boolean,
+    enforceInstalled: boolean,
+  ): HarnessChainEntry | null => {
+    const harness = getHarness(canonical);
+    if (!harness) {
+      if (reportIssues) {
+        const availableNames = listHarnesses()
+          .map((h) => `"${h.name}"`)
+          .join(", ");
+        issues.push({
+          requested: canonical,
+          reason: "unknown",
+          message: `Unknown agent harness "${canonical}" in AGENT_HARNESS; skipping it. Available harnesses: ${availableNames}.`,
+        });
+      }
+      return null;
+    }
+
+    const path = resolveEntryPath(harness, isPrimary, warnDeprecated);
+    const installed = options.isInstalled
+      ? options.isInstalled({ name: harness.name, path })
+      : isHarnessCliAvailable(path);
+
+    if (enforceInstalled && checkInstalled && !installed) {
+      if (reportIssues) {
+        const envKey = harness.name.toUpperCase().replace(/-/g, "_");
+        issues.push({
+          requested: canonical,
+          reason: "not-installed",
+          message: `Harness "${canonical}" is not installed (looked for "${path}"); skipping it. Install its CLI or set ${envKey}_CLI_PATH to the executable.`,
+        });
+      }
+      return null;
+    }
+
+    return { name: harness.name, harness, path, installed };
+  };
+
+  // Emit deprecation warnings for requested alias names (e.g. `gemini`),
+  // once per distinct alias, mirroring resolveHarness's behavior.
+  if (warnDeprecated !== false) {
+    const warnedAliases = new Set<string>();
+    for (const token of (raw ?? "").split(",")) {
+      const requested = token.trim();
+      const alias = requested ? HARNESS_ALIASES[requested] : undefined;
+      if (alias?.deprecated && alias.warning && !warnedAliases.has(requested)) {
+        warnedAliases.add(requested);
+        console.warn(`⚠️  ${alias.warning}`);
+      }
+    }
+  }
+
+  parsed.forEach((canonical, index) => {
+    const entry = buildEntry(canonical, index === 0, true, true);
+    if (entry) {
+      entries.push(entry);
+    }
+  });
+
+  if (entries.length === 0) {
+    if (parsed.every((canonical) => !getHarness(canonical))) {
+      // Nothing but unknown names: a configuration error, same as handing
+      // resolveHarness a single invalid name.
+      const availableNames = listHarnesses()
+        .map((h) => `"${h.name}"`)
+        .join(", ");
+      throw new Error(
+        `Unknown agent harness "${parsed.join(",")}" in AGENT_HARNESS. ` +
+          `Available harnesses: ${availableNames}. ` +
+          `Set AGENT_HARNESS to a comma-separated list of registered harnesses.`,
+      );
+    }
+    // Everything was dropped as not installed: keep the full parsed list so
+    // the spawn path reports the familiar "CLI not found" error instead of
+    // failing startup with an empty chain.
+    parsed.forEach((canonical, index) => {
+      const entry = buildEntry(canonical, index === 0, false, false);
+      if (entry) {
+        entries.push(entry);
+      }
+    });
+  }
+
+  return {
+    entries,
+    issues,
+    parsed,
+    multiHarness: parsed.length > 1,
+  };
+}

packages/agent-harness/src/index.ts

@@ -32,7 +32,23 @@ export {
 } from "./modes.js";
 
 // Registry
-export { registerHarness, getHarness, listHarnesses, HARNESS_ALIASES } from "./registry.js";
+export {
+  registerHarness,
+  getHarness,
+  listHarnesses,
+  HARNESS_ALIASES,
+  DEFAULT_HARNESS_NAME,
+} from "./registry.js";
+
+// Harness chain (comma-separated AGENT_HARNESS failover lists)
+export {
+  parseHarnessList,
+  resolveHarnessChain,
+  type HarnessChainEntry,
+  type HarnessChainIssue,
+  type HarnessChainOptions,
+  type ResolvedHarnessChain,
+} from "./harness-chain.js";
 
 // Prompt argument construction
 export { buildPromptArgs } from "./prompt-args.js";

packages/agent-harness/src/registry.ts

@@ -22,6 +22,9 @@ import {
 
 const registry = new Map<string, AgentHarness>();
 
+/** Default harness when no harness is configured (e.g. `AGENT_HARNESS` unset). */
+export const DEFAULT_HARNESS_NAME = "claude-code";
+
 /**
  * Alias map: alternate harness ids → canonical registered name.
  *

packages/agent-harness/src/resolver.ts

@@ -3,7 +3,8 @@
  *
  * Resolution order for harness name:
  *   1. `options.harnessName`
- *   2. `AGENT_HARNESS` environment variable
+ *   2. `AGENT_HARNESS` environment variable (a comma-separated list uses its
+ *      first entry — see `resolveHarnessChain` for full-chain resolution)
  *   3. Default to "claude-code" (backward compatible)
  *
  * Resolution order for executable path:
@@ -17,7 +18,7 @@
 import { execSync } from "child_process";
 import { accessSync, constants, existsSync } from "fs";
 import { resolve } from "path";
-import { getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js";
+import { DEFAULT_HARNESS_NAME, getHarness, HARNESS_ALIASES, listHarnesses } from "./registry.js";
 import type { AgentHarness, ResolvedHarness } from "./types.js";
 
 export interface HarnessResolutionOptions {
@@ -41,6 +42,10 @@ export interface HarnessResolutionOptions {
  * Resolve which harness to use and the CLI executable path to invoke.
  *
  * Harness name: `options.harnessName` → `AGENT_HARNESS` → `"claude-code"`.
+ * A comma-separated `AGENT_HARNESS` (e.g. `claude-code,codex`) names a
+ * failover chain; this resolver always picks the first (priority) entry so
+ * one-shot flows keep behaving as a single-harness run — worker-mode failover
+ * across the rest of the chain lives in {@link resolveHarnessChain}.
  * Aliases (e.g. `agy` / deprecated `gemini` → `antigravity`) are applied before
  * registry lookup; deprecated aliases emit a one-line console warning.
  *
@@ -65,10 +70,16 @@ export function resolveHarness(options?: HarnessResolutionOptions): ResolvedHarn
   const explicitHarnessName = options?.harnessName;
   let harnessName = explicitHarnessName;
   if (!harnessName) {
-    harnessName = env.AGENT_HARNESS;
+    // AGENT_HARNESS may name a comma-separated failover chain; env-driven
+    // (non-explicit) resolution always uses the first (priority) entry, kept
+    // raw so alias/deprecation warnings still fire for the requested name.
+    const firstEntry = env.AGENT_HARNESS?.split(",")[0]?.trim();
+    if (firstEntry) {
+      harnessName = firstEntry;
+    }
   }
   if (!harnessName) {
-    harnessName = "claude-code";
+    harnessName = DEFAULT_HARNESS_NAME;
   }
 
   const alias = HARNESS_ALIASES[harnessName];