diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index bc1a392764..55c75df299 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -259,6 +259,7 @@ request, and the OTLP export places it there. | `session.opened` | `{method, sessionId, requestedSessionId?, fallback?}` | `system` | | `attempt.settled` | `{attempt, outcome, failure?: {kind, detail}}` | `system` | | `run.settled` | `{outcome, failure?, durationMs, attempts}` | `system` | +| `harness.switched` | `{from, to, reason}` | `system` | - `outcome` is `ok`, `failed`, `stuck` or `cancelled`. - `failure.kind` is a key of `FAILURE_KINDS` in `src/failures.js`, or `null` @@ -274,6 +275,17 @@ request, and the OTLP export places it there. nor session/load`, or `session/ answered error ` with the JSON-RPC error's integer code; - no other field is allowed. +- `harness.switched` comes before the `attempt.started` of an attempt that + runs on another entry of the seat's run config than the one the seat was on + (`src/harness-plan.js`): + - `from` and `to` are entry labels, `harness/model` or + `harness/model@effort`, written the one canonical way and never equal; + - `reason` is a key of `SWITCH_REASONS`: `not_authenticated`, `quota` or + `provider_unsupported` (the entry above failed that way), `blocked` + (the entries above were skipped), `probe` (a probe moved back up the + list) or `operator` (an operator switched the seat); + - no other field is allowed. The attempt that follows starts a fresh + session. - `run.settled` is the last line of a complete record. Nothing is written after it. - If an attempt has no `attempt.settled`, the record says only that the next @@ -497,6 +509,10 @@ What's new or changed in `EVENT_TYPES`: `egress.denied` and `agent.retry`, `contextTokens` and `contextSize` on `usage`, and the `pod-member` value of `decidedByRole`. Lines without them are unchanged. + + A later revision, also at `v: 1`, added the kind `harness.switched` and the + failure kinds `not_authenticated`, `quota`, `provider_unsupported` and + `capacity`. Lines without them are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -549,7 +565,7 @@ operates it. the time. - **Every other kind is an event on the span current at the time.** That covers grants, credentials, claims, `task.done`, results, liveness, - `session.opened`, `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. + `session.opened`, `harness.switched`, `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. - **Span ids** are the first 8 bytes of SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to the entry that opens the span. The run span uses `0`. Exporting the same diff --git a/packages/seats/src/failures.js b/packages/seats/src/failures.js index 046a39c852..b3ff12fb1d 100644 --- a/packages/seats/src/failures.js +++ b/packages/seats/src/failures.js @@ -16,6 +16,12 @@ export const FAILURE_KINDS = Object.freeze({ cancelled: { retryable: false, meaning: 'the run was cancelled' }, credential_denied: { retryable: false, meaning: 'the run asked for a credential its grant does not allow' }, unsupported: { retryable: false, meaning: 'the CLI or its version is outside what this adapter supports' }, + // The harness cannot serve anyone right now. Retrying on the same harness is + // pointless; a run config's fallback moves to its next entry (harness-plan.js). + not_authenticated: { retryable: false, meaning: 'the CLI is not signed in, or its sign-in was rejected' }, + quota: { retryable: false, meaning: 'the account behind the CLI is out of quota or credits' }, + provider_unsupported: { retryable: false, meaning: 'the provider does not serve this model to this account' }, + capacity: { retryable: true, meaning: 'every entry the run config allows is at a host cap or held; it may run once one frees' }, }); export class RunFailure extends Error { diff --git a/packages/seats/src/harness-plan.js b/packages/seats/src/harness-plan.js new file mode 100644 index 0000000000..c6dfef4885 --- /dev/null +++ b/packages/seats/src/harness-plan.js @@ -0,0 +1,329 @@ +/** + * Which harness a seat runs a turn on, and what it does when that harness + * cannot serve. + * + * A seat declares a run config: a primary entry {harness, model, effort?} and + * an ordered fallback list of the same shape. `runTurnWithFallback` tries the + * active entry. When the attempt fails with one of FALLBACK_KINDS (the harness + * cannot serve anyone right now: not signed in, out of quota, the provider + * does not take this model), it moves to the next entry in the same turn, so + * the turn is still served. Every other failure is thrown unchanged and no + * switch happens. A content refusal in particular never switches: re-sending + * refused content through another provider is refusal-shopping. + * + * While a fallback is active, the next turn after the failed entry's cooldown + * starts from the primary again. That turn is the probe: if the primary + * answers, the seat is back on it. If it fails with a fallback kind or a + * retryable liveness kind (PROBE_CARRY_KINDS), the same turn carries on down + * the list. Any other failure, an unnamed one included, ends the turn there: + * an adapter's upstream refusal reaches this module as an unnamed error, and + * carrying it on would be the same refusal-shopping. Either way the cooldown + * restarts, so the next turn stays on the seat's own entry. + * + * An entry the host's policy blocks is skipped and never attempted. An entry + * the host cannot take right now (a per-model concurrency cap, a stop file; + * the caller's `acquire`) is skipped for this turn only. If no entry served, + * the turn fails visibly; nothing substitutes a harness the config does not + * list. + * + * Pure: state goes in and comes out as plain data, persisted by the caller, + * and time is passed in. Nothing here reads a file or the clock. + */ +import { RunFailure, failureKindOf } from './failures.js'; + +/** The failure kinds that move a turn to the next entry. Nothing else does. */ +export const FALLBACK_KINDS = Object.freeze(['not_authenticated', 'quota', 'provider_unsupported']); + +/** + * The failure kinds on which a failed probe carries on to the seat's own + * entry: the fallback kinds, and the retryable kinds that say the primary + * never produced an answer. An allowlist, so a kind added later ends a probe + * turn until someone decides otherwise. + */ +export const PROBE_CARRY_KINDS = Object.freeze([...FALLBACK_KINDS, 'stuck', 'timeout', 'exited']); + +/** How long a failed entry rests before a turn probes it again, per kind. */ +export const PROBE_COOLDOWN_MS = Object.freeze({ + quota: 15 * 60 * 1000, + not_authenticated: 30 * 60 * 1000, + provider_unsupported: 60 * 60 * 1000, +}); + +/** + * Why a seat's active entry changed, as `harness.switched` records it: + * a fallback kind, `blocked` (the active entry is blocked by policy), `probe` + * (a probe moved back up the list) and `operator` (`commonly agent use`). + */ +export const SWITCH_REASONS = Object.freeze([...FALLBACK_KINDS, 'blocked', 'probe', 'operator']); + +/** Fallback entries a config may list after its primary. */ +export const MAX_FALLBACK = 5; +/** Switches kept in state.history, newest last. */ +export const HISTORY_LIMIT = 20; + +const HARNESS = /^[a-z][a-z0-9-]{0,31}$/; +// No '@' and no whitespace: '@' separates the effort in the short form. +const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$/; +const EFFORT = /^[a-z]{1,16}$/; +const REASON = /^[A-Za-z0-9][A-Za-z0-9 ._:/#()-]{0,119}$/; + +const fail = (message) => { throw new Error(`harness plan: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); +}; + +/** + * One entry, as an object or the short form `harness/model[@effort]` + * (`claude/claude-sonnet-5-5@high`). The harness is everything before the first '/', + * so a model may itself contain '/' (`opencode/litellm/deepseek-flash-1017`). + */ +export const parseEntry = (value, where = 'entry') => { + let entry = value; + if (typeof value === 'string') { + const slash = value.indexOf('/'); + if (slash < 1) fail(`${where} must read harness/model[@effort], got ${JSON.stringify(value)}`); + const rest = value.slice(slash + 1); + const at = rest.lastIndexOf('@'); + entry = at === -1 + ? { harness: value.slice(0, slash), model: rest } + : { harness: value.slice(0, slash), model: rest.slice(0, at), effort: rest.slice(at + 1) }; + } + if (!isPlainObject(entry)) fail(`${where} must be an object or harness/model[@effort]`); + refuseUnknownKeys(entry, ['harness', 'model', 'effort'], `${where}.`); + if (typeof entry.harness !== 'string' || !HARNESS.test(entry.harness)) fail(`${where}.harness must match ${HARNESS}`); + if (typeof entry.model !== 'string' || !MODEL.test(entry.model)) fail(`${where}.model must match ${MODEL}`); + if (entry.effort !== undefined && (typeof entry.effort !== 'string' || !EFFORT.test(entry.effort))) { + fail(`${where}.effort must match ${EFFORT}, or be absent`); + } + return Object.freeze({ harness: entry.harness, model: entry.model, ...(entry.effort ? { effort: entry.effort } : {}) }); +}; + +export const entryLabel = (entry) => `${entry.harness}/${entry.model}${entry.effort ? `@${entry.effort}` : ''}`; + +/** + * `{harness, model, effort?, fallback?: [entry, …]}` → `{primary, fallback, chain, key}`. + * Refuses unknown fields, a fallback longer than MAX_FALLBACK and a repeated + * entry (an entry listed twice would be retried under another position). + */ +export const validateRunConfig = (config) => { + if (!isPlainObject(config)) fail('run config must be an object'); + refuseUnknownKeys(config, ['harness', 'model', 'effort', 'fallback'], 'config.'); + const primary = parseEntry({ + harness: config.harness, model: config.model, ...(config.effort !== undefined ? { effort: config.effort } : {}), + }, 'config'); + const list = config.fallback ?? []; + if (!Array.isArray(list)) fail('config.fallback must be an array'); + if (list.length > MAX_FALLBACK) fail(`config.fallback lists at most ${MAX_FALLBACK} entries`); + const fallback = list.map((item, index) => parseEntry(item, `config.fallback[${index}]`)); + const chain = [primary, ...fallback]; + const labels = chain.map(entryLabel); + const repeated = labels.find((label, index) => labels.indexOf(label) !== index); + if (repeated) fail(`config lists ${repeated} twice`); + return Object.freeze({ primary, fallback: Object.freeze(fallback), chain: Object.freeze(chain), key: labels.join(' > ') }); +}; + +/** + * Host policy: `{blocked?: [{harness?, model?, reason}], limits?: [{model, maxConcurrent}]}`. + * A blocked rule names a harness, a model or both, and matches an entry when + * every field it names is equal. The reason is a short operator word (a task + * id), recorded with each skip. A limit caps how many turns on this host run + * one model at once; it is enforced by the caller's `acquire`, which this + * module only consults. Absent or empty policy blocks and caps nothing. + */ +export const validatePolicy = (policy) => { + if (policy === undefined || policy === null) return Object.freeze({ blocked: Object.freeze([]), limits: Object.freeze([]) }); + if (!isPlainObject(policy)) fail('policy must be an object'); + refuseUnknownKeys(policy, ['blocked', 'limits'], 'policy.'); + const list = policy.blocked ?? []; + if (!Array.isArray(list)) fail('policy.blocked must be an array'); + const blocked = list.map((rule, index) => { + const where = `policy.blocked[${index}]`; + if (!isPlainObject(rule)) fail(`${where} must be an object`); + refuseUnknownKeys(rule, ['harness', 'model', 'reason'], `${where}.`); + if (rule.harness === undefined && rule.model === undefined) fail(`${where} names a harness, a model or both`); + if (rule.harness !== undefined && (typeof rule.harness !== 'string' || !HARNESS.test(rule.harness))) fail(`${where}.harness must match ${HARNESS}`); + if (rule.model !== undefined && (typeof rule.model !== 'string' || !MODEL.test(rule.model))) fail(`${where}.model must match ${MODEL}`); + if (typeof rule.reason !== 'string' || !REASON.test(rule.reason)) fail(`${where}.reason must match ${REASON}`); + return Object.freeze({ ...rule }); + }); + const capList = policy.limits ?? []; + if (!Array.isArray(capList)) fail('policy.limits must be an array'); + const limits = capList.map((limit, index) => { + const where = `policy.limits[${index}]`; + if (!isPlainObject(limit)) fail(`${where} must be an object`); + refuseUnknownKeys(limit, ['model', 'maxConcurrent'], `${where}.`); + if (typeof limit.model !== 'string' || !MODEL.test(limit.model)) fail(`${where}.model must match ${MODEL}`); + if (!Number.isInteger(limit.maxConcurrent) || limit.maxConcurrent < 1) fail(`${where}.maxConcurrent must be an integer >= 1`); + return Object.freeze({ model: limit.model, maxConcurrent: limit.maxConcurrent }); + }); + const models = limits.map((limit) => limit.model); + const twice = models.find((model, index) => models.indexOf(model) !== index); + if (twice) fail(`policy.limits lists ${twice} twice`); + return Object.freeze({ blocked: Object.freeze(blocked), limits: Object.freeze(limits) }); +}; + +/** The concurrency cap for this entry's model, or null. */ +export const limitFor = (entry, policy) => (policy?.limits || []).find((limit) => limit.model === entry.model) || null; + +/** The rule that blocks this entry, or null. */ +export const blockedBy = (entry, policy) => (policy?.blocked || []).find((rule) => ( + (rule.harness === undefined || rule.harness === entry.harness) + && (rule.model === undefined || rule.model === entry.model) +)) || null; + +/** State for a config that has never run: on the primary, nothing to probe. */ +export const initialState = (plan) => ({ + v: 1, configKey: plan.key, active: 0, since: null, reason: null, nextProbeAt: null, history: [], +}); + +// A persisted probe time as ms. An ISO string (a hand-edited or foreign +// state file) is read as its time; anything unreadable means the probe is due. +const probeTimeOf = (value) => { + if (Number.isFinite(value)) return value; + const parsed = typeof value === 'string' ? Date.parse(value) : NaN; + return Number.isFinite(parsed) ? parsed : null; +}; + +// State written by another version starts over. State for another config +// (the operator switched the seat) starts over on the new primary and keeps +// its history; the operator command records its own switch. +const stateFor = (plan, state) => { + if (!isPlainObject(state) || state.v !== 1) return initialState(plan); + if (state.configKey !== plan.key) { + return { ...initialState(plan), history: Array.isArray(state.history) ? state.history.slice(-HISTORY_LIMIT) : [] }; + } + const active = Number.isInteger(state.active) && state.active >= 0 && state.active < plan.chain.length ? state.active : 0; + return { + ...initialState(plan), ...state, active, nextProbeAt: probeTimeOf(state.nextProbeAt), + history: Array.isArray(state.history) ? state.history : [], + }; +}; + +/** + * The entry indices this turn tries, in order. From the active entry down; + * from the primary when a fallback is active and its probe is due. + */ +export const turnOrder = (plan, state, now) => { + const current = stateFor(plan, state); + const probing = current.active > 0 && (current.nextProbeAt === null || now >= current.nextProbeAt); + const start = probing ? 0 : current.active; + return { probing, order: plan.chain.map((_, index) => index).slice(start) }; +}; + +const iso = (ms) => new Date(ms).toISOString(); + +/** + * Run one turn on the plan. + * + * @param {object} args + * @param {ReturnType} args.plan + * @param {ReturnType} [args.policy] + * @param {object} [args.state] what the last turn returned; absent for a first run + * @param {number} args.now ms since the epoch + * @param {(entry, info: {index, fresh, probing, switch}) => Promise} args.attempt + * runs the turn on one entry. `fresh` is true when the entry is not the one + * the seat was on, so the caller starts a new session for it. `switch` is + * then `{from, to, reason}` (reason in SWITCH_REASONS) for the caller to + * write into that attempt's trace as `harness.switched`; null otherwise. + * @param {(entry, {limit}) => Promise<{release?: Function}|{unavailable: string}>} [args.acquire] + * asked before each attempt; `{unavailable}` skips the entry for this turn + * (its cap is full, the host holds it). `release` runs when the attempt + * settles. Default: every entry is available. + * @param {(error) => string|null} [args.classify] the failure kind; default failureKindOf + * @param {(skip: {entry, reason, blocked}) => void} [args.onSkip] per skipped entry + * @returns {Promise<{result, entry, index, state, attempts}>} + * On failure it rejects with the failure that ended the turn, carrying + * `harnessState` (persist it as on success) and `harnessAttempts`. + */ +export const runTurnWithFallback = async ({ + plan, policy, state, now, attempt, acquire = async () => ({}), classify = failureKindOf, onSkip = () => {}, +}) => { + if (!plan?.chain?.length) fail('runTurnWithFallback needs a validated plan'); + if (!Number.isFinite(now)) fail('runTurnWithFallback needs now'); + const current = stateFor(plan, state); + const { probing, order } = turnOrder(plan, current, now); + const was = plan.chain[current.active]; + const attempts = []; + let lastKind = null; + let probeKind = null; + let lastError = null; + const skip = (entry, reason, blocked) => { + attempts.push({ entry: entryLabel(entry), outcome: blocked ? 'blocked' : 'unavailable', reason }); + try { onSkip({ entry, reason, blocked }); } catch { /* a listener never fails a turn */ } + }; + + const cooldownOf = (kind) => PROBE_COOLDOWN_MS[kind] ?? PROBE_COOLDOWN_MS.quota; + const failWith = (error, next) => { + error.harnessState = next; + error.harnessAttempts = attempts; + return error; + }; + + for (const index of order) { + const entry = plan.chain[index]; + const rule = blockedBy(entry, policy); + if (rule) { skip(entry, rule.reason, true); continue; } + // eslint-disable-next-line no-await-in-loop + const slot = await acquire(entry, { limit: limitFor(entry, policy) }); + if (typeof slot?.unavailable === 'string') { skip(entry, slot.unavailable, false); continue; } + const fresh = index !== current.active; + // Why this attempt is not on the seat's entry: what failed above it, a + // probe moving up, or the entries above it skipped. + let reason = null; + if (fresh) reason = index < current.active ? 'probe' : (lastKind || 'blocked'); + try { + // eslint-disable-next-line no-await-in-loop + const result = await attempt(entry, { + index, fresh, probing, switch: fresh ? { from: was, to: entry, reason } : null, + }); + attempts.push({ entry: entryLabel(entry), outcome: 'ok' }); + const next = { ...current, active: index }; + if (fresh) { + const why = index < current.active ? 'recovered' : reason; + next.since = iso(now); + next.reason = why; + next.history = [...current.history, { at: iso(now), from: entryLabel(was), to: entryLabel(entry), reason: why }] + .slice(-HISTORY_LIMIT); + } + // Back on the primary there is nothing to probe. On a fallback, the + // next probe waits out the cooldown of what failed above it this turn; + // a turn that simply stayed put keeps the probe time it had. + if (index === 0) next.nextProbeAt = null; + else if (probeKind || fresh) next.nextProbeAt = now + cooldownOf(probeKind || lastKind); + return { result, entry, index, attempts, state: next }; + } catch (error) { + const kind = classify(error); + attempts.push({ entry: entryLabel(entry), outcome: 'failed', kind: kind || null }); + const probeFailed = probing && index < current.active; + if (probeFailed && probeKind === null) probeKind = kind || 'unnamed'; + // Not a reason to change harness: the turn fails as it would have with + // no fallback configured, and the seat stays where it was. A probe that + // failed only for want of an answer (PROBE_CARRY_KINDS) carries on to + // the seat's own entry; any other probe failure ends the turn too. + const carries = probeFailed ? PROBE_CARRY_KINDS.includes(kind) : FALLBACK_KINDS.includes(kind); + if (!carries) { + // A probe failed this turn, so the primary rests again whatever ended + // the turn; otherwise the next turn would probe it straight away. + const next = probeKind ? { ...current, nextProbeAt: now + cooldownOf(probeKind) } : current; + throw failWith(error, next); + } + // Only a failure at or below the seat's entry is a reason to move down + // from it; a probe's failures above it are the probe's own (probeKind). + if (FALLBACK_KINDS.includes(kind) && index >= current.active) lastKind = kind; + lastError = error; + } finally { + try { slot?.release?.(); } catch { /* releasing never fails a turn */ } + } + } + // Nothing served. Stay on the entry the seat was on and push the probe out, + // so the next turn does not go straight back to an entry that just failed. + const next = { ...current }; + if (current.active > 0 && (lastKind || probeKind)) next.nextProbeAt = now + cooldownOf(probeKind || lastKind); + const summary = attempts.map((a) => (a.outcome === 'failed' ? `${a.entry} ${a.kind || 'failed'}` : `${a.entry} ${a.outcome} (${a.reason})`)).join(', '); + if (lastError) throw failWith(lastError, next); + // Every entry was skipped: by policy alone it cannot run at all; with any + // entry merely unavailable it may run once a slot frees, and is retryable. + const onlyBlocked = attempts.every((a) => a.outcome === 'blocked'); + throw failWith(new RunFailure(onlyBlocked ? 'unsupported' : 'capacity', `no entry in the run config could run: ${summary}`), next); +}; diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 2612d8a91c..1c6dcf771b 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -7,6 +7,7 @@ import { createHash } from 'node:crypto'; import { isIP } from 'node:net'; import { FAILURE_KINDS } from '../failures.js'; +import { SWITCH_REASONS, entryLabel, parseEntry } from '../harness-plan.js'; export const RECORD_VERSION = 1; export const RECORD_SCHEMA = 'seats.record'; @@ -84,6 +85,8 @@ export const KINDS = Object.freeze({ 'session.opened': Object.freeze({ group: 'attempt', actor: 'system' }), 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + // The attempt that follows runs on another harness entry than the seat was on. + 'harness.switched': Object.freeze({ group: 'attempt', actor: 'system' }), 'tool.call': Object.freeze({ group: 'tool', actor: 'run' }), 'tool.update': Object.freeze({ group: 'tool', actor: 'run' }), 'credential.granted': Object.freeze({ group: 'credential', actor: 'system' }), @@ -401,6 +404,18 @@ const CHECKS = { } return data; }, + 'harness.switched': (data) => { + refuseUnknownKeys(data, ['from', 'to', 'reason'], 'harness.switched.'); + for (const key of ['from', 'to']) { + let entry = null; + try { entry = typeof data[key] === 'string' ? parseEntry(data[key]) : null; } catch { entry = null; } + // The canonical label only, so one entry is always written one way. + if (!entry || entryLabel(entry) !== data[key]) fail(`harness.switched.${key} must be an entry label harness/model[@effort]`); + } + if (data.from === data.to) fail('harness.switched.from and to must differ'); + if (!SWITCH_REASONS.includes(data.reason)) fail(`harness.switched.reason must be one of ${SWITCH_REASONS.join(', ')}`); + return data; + }, 'agent.retry': (data) => { refuseUnknownKeys(data, ['state', 'attempt', 'nextRetryAt', 'status', 'errorType'], 'agent.retry.'); if (!RETRY_STATES.includes(data.state)) fail(`agent.retry.state must be one of ${RETRY_STATES.join(', ')}`); diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index 229a7ca7f9..8358aa9490 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -35,6 +35,7 @@ export const EVENT_TYPES = Object.freeze([ 'attempt.settled', // { attempt, outcome, failure?: {kind, detail} } 'decision.ruled', // { decisionCardId, value, decidedBy, decidedByRole } a card ruling not tied to a parked tool call 'result.reported', // { status: 'done'|'partial'|'blocked'|'failed', refs?: [{type, id}] } + 'harness.switched', // { from, to, reason } entry labels harness/model[@effort]; reason in SWITCH_REASONS ]); const KNOWN = new Set(EVENT_TYPES); diff --git a/packages/seats/test/harness-plan.test.js b/packages/seats/test/harness-plan.test.js new file mode 100644 index 0000000000..9dd02cd743 --- /dev/null +++ b/packages/seats/test/harness-plan.test.js @@ -0,0 +1,357 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { FAILURE_KINDS, RunFailure } from '../src/failures.js'; +import { + FALLBACK_KINDS, HISTORY_LIMIT, MAX_FALLBACK, PROBE_CARRY_KINDS, PROBE_COOLDOWN_MS, SWITCH_REASONS, + entryLabel, initialState, parseEntry, runTurnWithFallback, turnOrder, validatePolicy, validateRunConfig, +} from '../src/harness-plan.js'; +import { appendEntry, KINDS } from '../src/record/format.js'; +import { EVENT_TYPES } from '../src/trace/events.js'; +import { adapterFailure } from '../src/upstream-refusal.js'; + +const NOW = Date.UTC(2026, 9, 26, 4, 0, 0); +const MIN = 60 * 1000; +const iso = (ms) => new Date(ms).toISOString(); + +// claude/Sonnet → claude/Haiku → opencode/DeepSeek. No codex/GPT entry: the +// fleet has no GPT access (D111), so no example chain falls back to one. +const CONFIG = { + harness: 'claude', + model: 'claude-sonnet-5-5', + effort: 'high', + fallback: ['claude/claude-haiku-5-5', 'opencode/litellm/deepseek-flash-1017'], +}; +const plan = validateRunConfig(CONFIG); +const [SONNET, HAIKU, DEEPSEEK] = plan.chain.map(entryLabel); + +/** + * An attempt function scripted per entry label: a value resolves, a kind + * throws a RunFailure of that kind, an Error throws as is. Records each call. + */ +const scripted = (script) => { + const calls = []; + const attempt = async (entry, info) => { + const label = entryLabel(entry); + calls.push({ label, ...info, switch: info.switch && { from: entryLabel(info.switch.from), to: entryLabel(info.switch.to), reason: info.switch.reason } }); + const step = script[label]; + if (step === undefined) throw new Error(`unscripted attempt on ${label}`); + if (step instanceof Error) throw step; + if (typeof step === 'string' && FAILURE_KINDS[step]) throw new RunFailure(step, `${label} said ${step}`); + return { text: `${label} answered` }; + }; + return { attempt, calls }; +}; + +const run = (script, extra = {}) => { + const { attempt, calls } = scripted(script); + return runTurnWithFallback({ plan, now: NOW, attempt, ...extra }) + .then((out) => ({ ...out, calls }), (error) => ({ error, calls })); +}; + +test('entries: object or short form, the harness before the first slash, the effort after the last @', () => { + assert.deepEqual(parseEntry('codex/gpt-6-luna@xhigh'), { harness: 'codex', model: 'gpt-6-luna', effort: 'xhigh' }); + assert.deepEqual(parseEntry('opencode/litellm/deepseek-flash-1017'), { harness: 'opencode', model: 'litellm/deepseek-flash-1017' }); + assert.deepEqual(parseEntry({ harness: 'claude', model: 'claude-haiku-5-5' }), { harness: 'claude', model: 'claude-haiku-5-5' }); + assert.equal(entryLabel(parseEntry('codex/gpt-6-luna@xhigh')), 'codex/gpt-6-luna@xhigh'); + for (const bad of ['codex', '/gpt', 'Codex/gpt', 'codex/gpt luna', 'codex/gpt@X-HIGH', 'codex/gpt@', { harness: 'codex', model: 'm', extra: 1 }, 7]) { + assert.throws(() => parseEntry(bad), /harness plan:/, JSON.stringify(bad)); + } +}); + +test('a run config refuses unknown fields, a long fallback and a repeated entry', () => { + assert.equal(plan.key, `${SONNET} > ${HAIKU} > ${DEEPSEEK}`); + assert.throws(() => validateRunConfig({ ...CONFIG, retries: 2 }), /unknown field config.retries/); + assert.throws(() => validateRunConfig({ ...CONFIG, fallback: Array.from({ length: MAX_FALLBACK + 1 }, (_, i) => `claude/m-${i}`) }), /at most 5/); + assert.throws(() => validateRunConfig({ ...CONFIG, fallback: ['claude/claude-sonnet-5-5@high'] }), /lists claude\/claude-sonnet-5-5@high twice/); + assert.throws(() => validateRunConfig({ harness: 'codex' }), /config.model/); + assert.deepEqual(validateRunConfig({ harness: 'claude', model: 'claude-haiku-5-5' }).fallback, []); +}); + +test('the primary answers: no switch, no fresh session, state unchanged', async () => { + const out = await run({ [SONNET]: 'ok' }); + assert.equal(out.index, 0); + assert.deepEqual(out.calls.map((c) => [c.label, c.fresh, c.switch]), [[SONNET, false, null]]); + assert.deepEqual(out.state, initialState(plan)); +}); + +for (const kind of FALLBACK_KINDS) { + test(`${kind} on the primary moves the same turn to the next entry, in a fresh session, and records why`, async () => { + const out = await run({ [SONNET]: kind, [HAIKU]: 'ok' }); + assert.equal(out.result.text, `${HAIKU} answered`); + assert.deepEqual(out.calls.map((c) => [c.label, c.fresh]), [[SONNET, false], [HAIKU, true]]); + assert.deepEqual(out.calls[1].switch, { from: SONNET, to: HAIKU, reason: kind }); + assert.equal(out.state.active, 1); + assert.equal(out.state.reason, kind); + assert.equal(out.state.since, iso(NOW)); + assert.equal(out.state.nextProbeAt, NOW + PROBE_COOLDOWN_MS[kind]); + assert.deepEqual(out.state.history, [{ at: iso(NOW), from: SONNET, to: HAIKU, reason: kind }]); + }); +} + +test('every other failure kind is thrown unchanged, with no switch, and the seat stays put', async () => { + for (const kind of Object.keys(FAILURE_KINDS).filter((k) => !FALLBACK_KINDS.includes(k))) { + const out = await run({ [SONNET]: kind, [HAIKU]: 'ok' }); + assert.equal(out.error?.kind, kind, kind); + assert.deepEqual(out.calls.map((c) => c.label), [SONNET], `${kind} must not reach the next entry`); + assert.deepEqual(out.error.harnessState, initialState(plan), kind); + } + const plain = await run({ [SONNET]: new Error('claude exited with code 1: something else'), [HAIKU]: 'ok' }); + assert.match(plain.error.message, /something else/); + assert.deepEqual(plain.calls.map((c) => c.label), [SONNET]); +}); + +test('a refusal never switches harness: the refused content is not sent to another provider', async () => { + const out = await run({ [SONNET]: 'refusal', [HAIKU]: 'ok', [DEEPSEEK]: 'ok' }); + assert.equal(out.error.kind, 'refusal'); + assert.deepEqual(out.calls.map((c) => c.label), [SONNET]); +}); + +test('down the whole chain: each fallback kind steps once, and the last entry serves', async () => { + const out = await run({ [SONNET]: 'quota', [HAIKU]: 'not_authenticated', [DEEPSEEK]: 'ok' }); + assert.equal(out.index, 2); + assert.deepEqual(out.calls.map((c) => c.switch), [null, { from: SONNET, to: HAIKU, reason: 'quota' }, { from: SONNET, to: DEEPSEEK, reason: 'not_authenticated' }]); + assert.deepEqual(out.attempts, [ + { entry: SONNET, outcome: 'failed', kind: 'quota' }, + { entry: HAIKU, outcome: 'failed', kind: 'not_authenticated' }, + { entry: DEEPSEEK, outcome: 'ok' }, + ]); +}); + +test('a policy-blocked entry is skipped and never attempted, and the skip is reported', async () => { + // DeepSeek is blocked until TASK-401 is lifted. + const policy = validatePolicy({ blocked: [{ model: 'litellm/deepseek-flash-1017', reason: 'TASK-401' }] }); + const skips = []; + const out = await run({ [SONNET]: 'quota', [HAIKU]: 'quota', [DEEPSEEK]: 'ok' }, { policy, onSkip: (s) => skips.push([entryLabel(s.entry), s.reason, s.blocked]) }); + assert.deepEqual(out.calls.map((c) => c.label), [SONNET, HAIKU]); + assert.equal(out.error.kind, 'quota'); + assert.deepEqual(skips, [[DEEPSEEK, 'TASK-401', true]]); + assert.deepEqual(out.error.harnessAttempts.at(-1), { entry: DEEPSEEK, outcome: 'blocked', reason: 'TASK-401' }); +}); + +test('a blocked primary: the turn starts on the next entry, and says the entries above were skipped', async () => { + const policy = validatePolicy({ blocked: [{ model: 'claude-sonnet-5-5', reason: 'TASK-390' }] }); + const out = await run({ [HAIKU]: 'ok' }, { policy }); + assert.deepEqual(out.calls.map((c) => [c.label, c.switch?.reason ?? null]), [[HAIKU, 'blocked']]); + assert.equal(out.state.active, 1); +}); + +test('every entry blocked or down: the turn fails visibly, and nothing outside the config is tried', async () => { + const allBlocked = validatePolicy({ blocked: [{ model: 'claude-sonnet-5-5', reason: 'TASK-390' }, { harness: 'claude', reason: 'TASK-391' }, { harness: 'opencode', reason: 'TASK-401' }] }); + const blocked = await run({}, { policy: allBlocked }); + assert.equal(blocked.error.kind, 'unsupported'); + assert.equal(blocked.error.retryable, false); + assert.match(blocked.error.message, /no entry in the run config could run: claude\/claude-sonnet-5-5@high blocked \(TASK-390\)/); + assert.deepEqual(blocked.calls, []); + + const down = await run({ [SONNET]: 'quota', [HAIKU]: 'quota', [DEEPSEEK]: 'provider_unsupported' }); + assert.equal(down.error.kind, 'provider_unsupported'); + assert.deepEqual(down.calls.map((c) => c.label), [SONNET, HAIKU, DEEPSEEK]); + // Nothing served: the seat stays on the entry it was on. + assert.deepEqual(down.error.harnessState, initialState(plan)); +}); + +test('an entry at its host cap is skipped for this turn only; a slot taken is released', async () => { + const policy = validatePolicy({ limits: [{ model: 'claude-haiku-5-5', maxConcurrent: 2 }] }); + const asked = []; + let released = 0; + const acquire = async (entry, { limit }) => { + asked.push([entryLabel(entry), limit?.maxConcurrent ?? null]); + if (entry.model === 'claude-haiku-5-5') return { unavailable: 'claude-haiku-5-5 at its cap (2)' }; + return { release: () => { released += 1; } }; + }; + const out = await run({ [SONNET]: 'quota', [DEEPSEEK]: 'ok' }, { policy, acquire }); + assert.deepEqual(asked, [[SONNET, null], [HAIKU, 2], [DEEPSEEK, null]]); + assert.deepEqual(out.calls.map((c) => c.label), [SONNET, DEEPSEEK]); + assert.equal(released, 2); + assert.deepEqual(out.attempts[1], { entry: HAIKU, outcome: 'unavailable', reason: 'claude-haiku-5-5 at its cap (2)' }); + + // Nothing could run, only for want of a slot: retryable, not unsupported. + const full = await run({}, { acquire: async () => ({ unavailable: 'host stop file' }) }); + assert.equal(full.error.kind, 'capacity'); + assert.equal(full.error.retryable, true); +}); + +test('policy refuses malformed rules and limits', () => { + assert.throws(() => validatePolicy({ blocked: [{ reason: 'TASK-401' }] }), /names a harness, a model or both/); + assert.throws(() => validatePolicy({ blocked: [{ model: 'm', reason: '' }] }), /reason must match/); + assert.throws(() => validatePolicy({ limits: [{ model: 'claude-opus-5-5', maxConcurrent: 0 }] }), /maxConcurrent/); + assert.throws(() => validatePolicy({ limits: [{ model: 'm', maxConcurrent: 1 }, { model: 'm', maxConcurrent: 2 }] }), /lists m twice/); + assert.throws(() => validatePolicy({ allow: [] }), /unknown field policy.allow/); + assert.deepEqual(validatePolicy(undefined), { blocked: [], limits: [] }); +}); + +test('on a fallback, the next turn stays put until the cooldown, then probes the primary and returns to it', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const before = await runTurnWithFallback({ + plan, state: first.state, now: NOW + PROBE_COOLDOWN_MS.quota - 1, attempt: scripted({ [HAIKU]: 'ok' }).attempt, + }); + assert.equal(before.index, 1); + assert.equal(before.state.nextProbeAt, first.state.nextProbeAt, 'a turn that stays put keeps its probe time'); + + assert.deepEqual(turnOrder(plan, first.state, NOW + PROBE_COOLDOWN_MS.quota), { probing: true, order: [0, 1, 2] }); + const probe = scripted({ [SONNET]: 'ok' }); + const back = await runTurnWithFallback({ plan, state: before.state, now: NOW + PROBE_COOLDOWN_MS.quota, attempt: probe.attempt }); + assert.equal(back.index, 0); + assert.deepEqual(probe.calls[0].switch, { from: HAIKU, to: SONNET, reason: 'probe' }); + assert.equal(back.state.nextProbeAt, null); + assert.deepEqual(back.state.history.at(-1), { at: iso(NOW + PROBE_COOLDOWN_MS.quota), from: HAIKU, to: SONNET, reason: 'recovered' }); +}); + +test('a probe that fails for want of an answer carries on to the seat\'s entry, and the cooldown restarts', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const at = NOW + PROBE_COOLDOWN_MS.quota; + // The probe's own failure sets the next cooldown. A liveness kind (stuck, + // timeout, exited) names no fallback kind and rests the shortest, quota's. + for (const [failure, cooldown] of [ + ['quota', PROBE_COOLDOWN_MS.quota], + ['provider_unsupported', PROBE_COOLDOWN_MS.provider_unsupported], + ['stuck', PROBE_COOLDOWN_MS.quota], + ['timeout', PROBE_COOLDOWN_MS.quota], + ['exited', PROBE_COOLDOWN_MS.quota], + ]) { + const turn = scripted({ [SONNET]: failure, [HAIKU]: 'ok' }); + const out = await runTurnWithFallback({ plan, state: first.state, now: at, attempt: turn.attempt }); + assert.equal(out.index, 1, String(failure)); + // Back on its own entry: same session, no switch. + assert.deepEqual(turn.calls.map((c) => [c.label, c.fresh]), [[SONNET, true], [HAIKU, false]]); + assert.equal(out.state.nextProbeAt, at + cooldown, String(failure)); + } + assert.deepEqual(PROBE_CARRY_KINDS, [...FALLBACK_KINDS, 'stuck', 'timeout', 'exited']); +}); + +test('any other probe failure ends the turn on the primary, and the primary rests again', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const at = NOW + PROBE_COOLDOWN_MS.quota; + // The adapter's own shape for an upstream refusal: a plain Error with no + // kind, exactly as adapters raise it. It must never reach HAIKU. + const upstream = adapterFailure('claude', 'This content was flagged for possible cybersecurity risk.', { exitCode: 1 }); + for (const failure of [upstream, new Error('claude exited with code 1: unnamed'), 'refusal', 'cancelled', 'credential_denied', 'max_tokens', 'max_turn_requests', 'protocol']) { + const turn = scripted({ [SONNET]: failure, [HAIKU]: 'ok' }); + const out = await runTurnWithFallback({ plan, state: first.state, now: at, attempt: turn.attempt }).catch((e) => e); + assert.ok(out instanceof Error, String(failure)); + assert.deepEqual(turn.calls.map((c) => c.label), [SONNET], String(failure)); + // The seat stays on its entry, and the next turn does not probe again at once. + assert.equal(out.harnessState.active, 1, String(failure)); + assert.equal(out.harnessState.nextProbeAt, at + PROBE_COOLDOWN_MS.quota, String(failure)); + assert.equal(turnOrder(plan, out.harnessState, at + 1).probing, false, String(failure)); + } +}); + +test('a failed probe restarts the cooldown even when the seat\'s own entry then fails', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const at = NOW + PROBE_COOLDOWN_MS.quota; + for (const own of ['stuck', 'refusal', new Error('claude exited with code 1: unnamed')]) { + const turn = scripted({ [SONNET]: 'provider_unsupported', [HAIKU]: own }); + const out = await runTurnWithFallback({ plan, state: first.state, now: at, attempt: turn.attempt }).catch((e) => e); + assert.deepEqual(turn.calls.map((c) => c.label), [SONNET, HAIKU], String(own)); + assert.equal(out.harnessState.active, 1, String(own)); + assert.equal(out.harnessState.nextProbeAt, at + PROBE_COOLDOWN_MS.provider_unsupported, String(own)); + assert.equal(turnOrder(plan, out.harnessState, at + 1).probing, false, String(own)); + } +}); + +test('a served turn after a failed probe rests the primary by the probe\'s failure, not the seat\'s', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const at = NOW + PROBE_COOLDOWN_MS.quota; + const out = await runTurnWithFallback({ + plan, state: first.state, now: at, + attempt: scripted({ [SONNET]: 'provider_unsupported', [HAIKU]: 'quota', [DEEPSEEK]: 'ok' }).attempt, + }); + assert.equal(out.index, 2); + assert.equal(out.state.nextProbeAt, at + PROBE_COOLDOWN_MS.provider_unsupported); +}); + +test('a persisted probe time that is not a number is read, not trusted as is', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const past = { ...first.state, nextProbeAt: iso(NOW - 60 * MIN) }; + assert.equal(turnOrder(plan, past, NOW).probing, true); + const future = { ...first.state, nextProbeAt: iso(NOW + 60 * MIN) }; + assert.equal(turnOrder(plan, future, NOW).probing, false); + assert.equal(turnOrder(plan, { ...first.state, nextProbeAt: 'soon' }, NOW).probing, true); + // A served turn on the fallback writes it back as a number. + const stayed = await runTurnWithFallback({ plan, state: future, now: NOW, attempt: scripted({ [HAIKU]: 'ok' }).attempt }); + assert.equal(stayed.state.nextProbeAt, NOW + 60 * MIN); +}); + +test('a failed probe sets the next probe by the primary\'s failure, not by an entry between', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'not_authenticated', [DEEPSEEK]: 'ok' }); + assert.equal(first.state.active, 2); + const at = first.state.nextProbeAt; + const out = await runTurnWithFallback({ + plan, state: first.state, now: at, + attempt: scripted({ [SONNET]: 'quota', [HAIKU]: 'provider_unsupported', [DEEPSEEK]: 'ok' }).attempt, + }); + assert.equal(out.index, 2); + assert.equal(out.state.nextProbeAt, at + PROBE_COOLDOWN_MS.quota); +}); + +test('a switch names its own reason: a move up is a probe, a move down past a skip is blocked', async () => { + // The seat is on DEEPSEEK; the primary fails its probe and HAIKU, above the + // seat, answers. Nothing above DEEPSEEK failed from DEEPSEEK's point of view. + const onThird = await run({ [SONNET]: 'quota', [HAIKU]: 'not_authenticated', [DEEPSEEK]: 'ok' }); + const up = scripted({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const moved = await runTurnWithFallback({ plan, state: onThird.state, now: onThird.state.nextProbeAt, attempt: up.attempt }); + assert.deepEqual(up.calls.map((c) => c.switch), [ + { from: DEEPSEEK, to: SONNET, reason: 'probe' }, { from: DEEPSEEK, to: HAIKU, reason: 'probe' }, + ]); + assert.equal(moved.state.reason, 'recovered'); + // The seat is on HAIKU; the probe fails, HAIKU is at its cap, DEEPSEEK + // serves. HAIKU did not fail with quota; it was skipped. + const onSecond = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const down = scripted({ [SONNET]: 'quota', [DEEPSEEK]: 'ok' }); + const acquire = async (entry) => (entryLabel(entry) === HAIKU ? { unavailable: 'at its cap' } : {}); + const past = await runTurnWithFallback({ plan, state: onSecond.state, now: onSecond.state.nextProbeAt, attempt: down.attempt, acquire }); + assert.deepEqual(down.calls.map((c) => c.switch?.reason), ['probe', 'blocked']); + assert.equal(past.state.reason, 'blocked'); +}); + +test('a probe that fails with not_authenticated waits out that kind\'s cooldown, not the one it fell back for', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const at = NOW + PROBE_COOLDOWN_MS.quota; + const out = await runTurnWithFallback({ plan, state: first.state, now: at, attempt: scripted({ [SONNET]: 'not_authenticated', [HAIKU]: 'ok' }).attempt }); + assert.equal(out.state.nextProbeAt, at + PROBE_COOLDOWN_MS.not_authenticated); +}); + +test('state for another config starts on its primary and keeps history; a newer version starts over', async () => { + const first = await run({ [SONNET]: 'quota', [HAIKU]: 'ok' }); + const other = validateRunConfig({ harness: 'claude', model: 'claude-opus-5-5', fallback: ['claude/claude-haiku-5-5'] }); + const turn = scripted({ 'claude/claude-opus-5-5': 'ok' }); + const out = await runTurnWithFallback({ plan: other, state: first.state, now: NOW + 1, attempt: turn.attempt }); + assert.equal(out.index, 0); + assert.equal(turn.calls[0].fresh, false); + assert.equal(out.state.configKey, other.key); + assert.deepEqual(out.state.history, first.state.history); + const fromFuture = await runTurnWithFallback({ plan, state: { ...first.state, v: 2 }, now: NOW, attempt: scripted({ [SONNET]: 'ok' }).attempt }); + assert.deepEqual(fromFuture.state, initialState(plan)); +}); + +test('history keeps the newest switches only', async () => { + let state; + for (let i = 0; i < HISTORY_LIMIT + 5; i += 1) { + const now = NOW + i * 2 * 60 * MIN; + // eslint-disable-next-line no-await-in-loop + const out = await runTurnWithFallback({ + plan, state, now, attempt: scripted(i % 2 ? { [SONNET]: 'ok' } : { [SONNET]: 'quota', [HAIKU]: 'ok' }).attempt, + }); + state = out.state; + } + assert.equal(state.history.length, HISTORY_LIMIT); +}); + +test('harness.switched is a trace event and a v1 record kind, validated', () => { + assert.ok(EVENT_TYPES.includes('harness.switched')); + assert.deepEqual(KINDS['harness.switched'], { group: 'attempt', actor: 'system' }); + assert.deepEqual(SWITCH_REASONS, ['not_authenticated', 'quota', 'provider_unsupported', 'blocked', 'probe', 'operator']); + const at = (data) => JSON.parse(appendEntry(null, { + runId: 'run-hp-1', ts: iso(NOW), kind: 'harness.switched', data, + }, { runActor: { type: 'agent', id: 'u-1' } })); + const line = at({ from: SONNET, to: HAIKU, reason: 'quota' }); + assert.deepEqual(line.data, { from: SONNET, to: HAIKU, reason: 'quota' }); + assert.deepEqual(line.actor, { type: 'system', id: 'seats' }); + for (const reason of SWITCH_REASONS) at({ from: SONNET, to: DEEPSEEK, reason }); + assert.throws(() => at({ from: SONNET, to: HAIKU, reason: 'refusal' }), /harness.switched.reason/); + assert.throws(() => at({ from: SONNET, to: SONNET, reason: 'quota' }), /must differ/); + assert.throws(() => at({ from: 'codex', to: HAIKU, reason: 'quota' }), /harness.switched.from/); + assert.throws(() => at({ from: { harness: 'codex', model: 'm' }, to: HAIKU, reason: 'quota' }), /harness.switched.from/); + assert.throws(() => at({ from: SONNET, to: HAIKU, reason: 'quota', detail: 'x' }), /unknown field harness.switched.detail/); +});