diff --git a/.github/workflows/seats-compat-nightly.yml b/.github/workflows/seats-compat-nightly.yml
new file mode 100644
index 0000000000..f9e38f6bf6
--- /dev/null
+++ b/.github/workflows/seats-compat-nightly.yml
@@ -0,0 +1,89 @@
+name: Seats compatibility nightly
+
+# Runs each pinned agent CLI (packages/seats/compat/targets.json) through the
+# compatibility suite against a local fake provider: no model provider, no
+# real key. A failing check is a result, not a job failure; the job fails
+# only when the matrix itself could not be produced.
+
+on:
+ schedule:
+ - cron: '23 9 * * *'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: seats-compat-nightly
+ cancel-in-progress: false
+
+jobs:
+ matrix:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ # The runner context is not available in job-level env, so the
+ # job-local paths are set here for every later step.
+ - name: Choose job-local paths
+ run: |
+ echo "COMPAT_PREFIX=$RUNNER_TEMP/compat-bin" >> "$GITHUB_ENV"
+ echo "COMPAT_OUT=$RUNNER_TEMP/compat-out" >> "$GITHUB_ENV"
+
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+
+ - uses: actions/setup-node@v4
+ with:
+ node-version: 22
+
+ # Aider (aider-chat) supports python 3.10 to 3.12 only.
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+
+ # Job-local prefix only: release assets are sha256-checked before they
+ # are unpacked, npm packages install with --ignore-scripts, pip packages
+ # install from wheels into a venv under the prefix. A CLI that
+ # fails to install shows up in the matrix as "binary not installed".
+ - name: Install pinned CLIs
+ continue-on-error: true
+ run: node packages/seats/compat/install.js --targets packages/seats/compat/targets.json --prefix "$COMPAT_PREFIX"
+
+ - name: Run the compatibility matrix
+ run: node packages/seats/compat/run.js --targets packages/seats/compat/targets.json --prefix "$COMPAT_PREFIX" --out "$COMPAT_OUT"
+
+ - name: Upload results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: seats-compat
+ path: ${{ env.COMPAT_OUT }}
+ if-no-files-found: error
+ retention-days: 30
+
+ publish:
+ # Disabled on purpose: publishing needs a maintainer to enable GitHub Pages
+ # for this repository (Settings → Pages → Source: GitHub Actions). Flip this
+ # to `if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'`
+ # once Pages is on.
+ if: false
+ needs: matrix
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ pages: write
+ id-token: write
+ environment:
+ name: github-pages
+ url: ${{ steps.deploy.outputs.page_url }}
+ steps:
+ - uses: actions/download-artifact@v4
+ with:
+ name: seats-compat
+ path: site
+ - uses: actions/upload-pages-artifact@v3
+ with:
+ path: site
+ - id: deploy
+ uses: actions/deploy-pages@v4
diff --git a/packages/seats/README.md b/packages/seats/README.md
index 522df333f3..81c25d1ce8 100644
--- a/packages/seats/README.md
+++ b/packages/seats/README.md
@@ -24,3 +24,24 @@ It is private and is not published on its own. `@commonlyai/cli` depends on it a
The extraction plan is D62, extract-in-place.
**Run record.** The trace store writes each run as a versioned, hash-chained JSONL record. It covers who granted which agent what access for which task, who approved each call, what was attempted, and the result. A tool outside this package can read, verify and export it to OTLP. The format is specified in [`docs/record-v1.md`](docs/record-v1.md), and a generated sample is in [`docs/examples/`](docs/examples/).
+
+## Compatibility matrix
+
+`compat/` runs each pinned CLI in `compat/targets.json` through a suite of checks (`compat/checks.js`) against the local fake provider in `test-support/`, and writes `results.json` plus a self-contained `index.html`:
+
+```
+node compat/install.js --targets compat/targets.json --prefix
# pinned binaries, job-local
+node compat/run.js --targets compat/targets.json --prefix --out
+```
+
+A target is data: `{ cli, version, adapter: 'module#export', binary?, enabled, requires?, probeOptions?, limits? }`. The export is an adapter or a factory taking `options`. A target whose adapter module is absent skips every check. `.github/workflows/seats-compat-nightly.yml` runs it nightly and uploads the output.
+
+- `probeOptions` builds a second instance of a factory export for the checks that read the agent itself. The ACP targets pass `{ watchProvider: false }`, so `retry-status` sees what the agent streams while it retries a 5xx, with no provider watch ending the turn first. `retry-status-surfaced` then asks whether the adapter's trace carried those reports.
+- `binary` names one install recipe: `release` (a per-platform tarball with its sha256), `npm` (installed with `--ignore-scripts`), or `pip` (an exact `name==version`, installed from wheels into a venv under the prefix; the interpreter is `python3` or `COMPAT_PYTHON`).
+- CLIs that run one turn per process in their own headless mode (copilot, qwen-code, cline, aider) share `src/adapters/headless.js`: a `launch` that returns argv and env, a `read` per stdout line, and the process group, deadline, cancellation and trace done once.
+- A CLI that cannot be pointed at the run's OpenAI- or Anthropic-compatible provider (gemini, cursor, amp) still gets a row. Its adapter (`src/adapters/provider-unsupported.js`) detects the pinned binary and fails every turn as `provider_unsupported` with the reason, and every turn check reports that failure, never a skip.
+- `limits: { : { note, fixedIn? } }` marks a failure that belongs to that CLI version, not to the adapter. The row keeps `status: 'fail'` and gains `limit`; the page shows it as a limit with the version that fixes it, and `totals.fail` counts only failures no limit explains. A declared limit that stops reproducing shows as a pass that says so.
+
+### Upstream bugs
+
+`compat/upstream.json` lists each known upstream bug with what we do about it: `{ id, cli, versions, bug, status, workaround: { name, ref, where, tests[] }, matrix?: { target, check }, references[] }`. We do not wait on upstream, so `status` is `"worked around"` or `"open"`, and `compat/upstream.js` enforces the first: a row reads as worked around only when its workaround names the pushed commit it lives on (`ref`, a full sha) and at least one test there that proves it. A row that claims more is shown as open, with the reason. An open row names its planned workaround. Upstream issue links go in `references`, never in place of a workaround. A row with `matrix` shows what that check did on this run beside the claim. `run.js` reads the file next to the targets, or the one `--upstream` names.
diff --git a/packages/seats/compat/checks.js b/packages/seats/compat/checks.js
new file mode 100644
index 0000000000..92e5060cb2
--- /dev/null
+++ b/packages/seats/compat/checks.js
@@ -0,0 +1,365 @@
+/**
+ * The compatibility suite. Each check is
+ * { id, title, run(target, env) → { status: 'pass'|'fail'|'skip', detail, ms? } }
+ * where `env` is the per-target harness (compat/harness.js). Checks share turns
+ * through env.turn(name, …) so a fresh PONG is paid for once, not five times.
+ *
+ * Every check runs under its own timeout (runCheck); a check that never
+ * settles is reported as a failure, and its open turns are cancelled.
+ */
+import { isNamedFailure } from './harness.js';
+
+const pass = (detail) => ({ status: 'pass', detail });
+const fail = (detail) => ({ status: 'fail', detail });
+const skip = (detail) => ({ status: 'skip', detail });
+
+const PONG = 'PONG';
+const RESUMED = 'PONG-2';
+
+// Prompts carry a fake-provider directive, so a real CLI's own system prompt
+// cannot change what the provider answers.
+const freshTurn = (env) => env.turn('fresh', `Reply with the single word ${PONG}. [[reply:${PONG}]]`);
+const resumedTurn = async (env) => {
+ const first = await freshTurn(env);
+ const sessionId = first.ok ? first.result.newSessionId : null;
+ if (!sessionId) return null;
+ return env.turn('resumed', `Reply with the single word ${RESUMED}. [[reply:${RESUMED}]]`, { sessionId });
+};
+const refusalTurn = (env) => env.turn('refusal', 'Answer this. [[refuse]]');
+// A provider watch that stops the CLI at the first refusal on the wire names
+// it refusal_retry_loop; both kinds are final and never retried.
+const REFUSAL_KINDS = new Set(['refusal', 'refusal_retry_loop']);
+const upstreamTurn = (env) => env.turn('upstream-5xx', 'Answer this. [[fail:503]]');
+const stallTurn = (env) => env.turn('stall', 'Answer this. [[stall]]', { timeoutMs: env.timeouts.stallMs });
+const ALL_TURNS = [freshTurn, resumedTurn, refusalTurn, upstreamTurn, stallTurn];
+// The same 503, on the probe adapter (the target's probeOptions, e.g. no
+// provider watch), held open for a window so the agent's own retries happen.
+const retryProbeTurn = (env) => env.probeTurn?.('retry-5xx', 'Answer this. [[fail:503]]', { timeoutMs: env.timeouts.retryWindowMs }) ?? null;
+
+// Trace events an adapter emits on its own account rather than from what the
+// agent streamed; they say nothing about whether a retry report got through.
+const LIFECYCLE_EVENT = /^(?:run\.|session\.|turn\.|process\.|permission\.|liveness\.|credential\.)/;
+// ACP update kinds that publish session state (commands, mode, config,
+// context size). Agents send them whenever they like, and goose 1.56.0 sends
+// two right as its first request goes out, so they never count as a report
+// about a retry; `usage` is what the adapter maps usage_update to.
+const SESSION_STATE = new Set(['available_commands_update', 'current_mode_update', 'config_option_update', 'usage_update']);
+const SESSION_STATE_EVENT = new Set(['usage']);
+const SAMPLE_CHARS = 600;
+
+const tally = (records) => {
+ const kinds = {};
+ const metaKeys = new Set();
+ for (const { msg } of records) {
+ const update = msg.params?.update || {};
+ const kind = update.sessionUpdate || 'unknown';
+ kinds[kind] = (kinds[kind] || 0) + 1;
+ for (const name of Object.keys(update._meta || {})) metaKeys.add(name);
+ }
+ return { kinds, metaKeys: [...metaKeys].sort() };
+};
+
+/**
+ * What a recorded ACP turn shows about retries: how often the provider was
+ * asked, what the agent streamed while it was still retrying, and what of
+ * that reached the trace. Reads only the tap, so it holds for any ACP agent.
+ *
+ * "While retrying" is placed by the harness stamp (provider requests made so
+ * far), not inferred from a kind name: an update counts when the agent had
+ * already asked the provider twice and asked again afterwards. The first
+ * request is the turn starting, and start-of-turn bookkeeping lands beside
+ * it; what an agent sends once it has given up (its error text) lands after
+ * the last. Both are reported, as "outside the retries", never as progress.
+ */
+export const readRetryProbe = (turn, { key = '' } = {}) => {
+ const wire = Array.isArray(turn?.wire) ? turn.wire : [];
+ const prompt = wire.find((w) => w.dir === 'out' && w.msg?.method === 'session/prompt');
+ const at = prompt ? wire.indexOf(prompt) : -1;
+ const after = at === -1 ? [] : wire.slice(at + 1);
+ const total = (turn?.requests || []).length;
+ const inWindow = (record) => Number.isFinite(record.requests) && record.requests >= 2 && record.requests < total;
+ const all = after.filter((w) => w.dir === 'in' && w.msg?.method === 'session/update');
+ const during = all.filter((w) => inWindow(w) && !SESSION_STATE.has(w.msg.params?.update?.sessionUpdate));
+ const answer = prompt ? after.find((w) => w.dir === 'in' && w.msg?.id === prompt.msg.id && !w.msg.method) : null;
+ const { kinds, metaKeys } = tally(during);
+ const rest = tally(all.filter((w) => !during.includes(w)));
+ const promptEvent = (turn?.events || []).findIndex((e) => e.type === 'turn.prompt_sent');
+ const surfaced = promptEvent === -1 ? []
+ : turn.events.slice(promptEvent + 1).filter((e) => !LIFECYCLE_EVENT.test(e.type) && !SESSION_STATE_EVENT.has(e.type) && inWindow(e));
+ const scrub = (text) => (key ? text.split(key).join('') : text);
+ return {
+ tapped: !!turn?.tapped,
+ promptSent: !!prompt,
+ failedRequests: (turn?.requests || []).filter((r) => r.directive === 'fail').length,
+ updates: during.length,
+ kinds,
+ metaKeys,
+ otherKinds: rest.kinds,
+ sample: during[0] ? scrub(JSON.stringify(during[0].msg.params)).slice(0, SAMPLE_CHARS) : null,
+ answerMetaKeys: Object.keys(answer?.msg?.result?._meta || answer?.msg?.error?.data?._meta || {}).sort(),
+ surfaced: surfaced.map((e) => e.type),
+ };
+};
+
+const listKinds = (kinds) => Object.entries(kinds).map(([kind, n]) => `${kind}×${n}`).join(', ');
+const needsProbe = async (env, then) => needsBinary(env, async () => {
+ const turn = await retryProbeTurn(env);
+ if (!turn) return skip('the target declares no probeOptions, so the agent is only seen through its adapter');
+ const probe = readRetryProbe(turn, { key: env.provider.apiKey });
+ if (!probe.tapped) return skip('the adapter took no spawn hook, so the agent\'s own stream was not recorded');
+ if (!probe.promptSent) return skip(`no session/prompt was sent (${turn.ok ? 'the turn succeeded' : describeError(turn.error)})`);
+ return then(turn, probe);
+});
+
+const describeError = (error) => (isNamedFailure(error)
+ ? `RunFailure ${error.kind}: ${error.detail || error.message}`
+ : `unnamed ${error?.name || typeof error}: ${String(error?.message ?? error)}`);
+const quote = (text, limit = 160) => JSON.stringify(String(text ?? '').slice(0, limit));
+
+// The turn checks need a binary; without one they would only repeat detect's
+// failure in five more confusing words. They also need a CLI that can reach
+// the run's provider: one that cannot fails every turn check with its named
+// reason, because a refusal, a 5xx or a stall it never saw proves nothing,
+// and provider-5xx or stall would otherwise pass on the instant failure.
+const needsBinary = async (env, then) => {
+ const { found } = await env.detect();
+ if (!found) return skip('detect() found no binary');
+ if (env.adapter?.supportsProvider === false) {
+ const turn = await freshTurn(env);
+ return fail(turn.ok
+ ? 'the adapter declares it cannot reach the provider, yet a turn succeeded'
+ : `cannot reach the run's provider: ${describeError(turn.error)}`);
+ }
+ return then();
+};
+
+export const CHECKS = [
+ {
+ id: 'detect',
+ title: 'detect() finds the binary and reports a version',
+ async run(target, env) {
+ const { found, error } = await env.detect();
+ if (error) return fail(`detect() threw: ${describeError(error)}`);
+ if (!found) return fail(env.binaryPath ? `detect() returned null (expected ${target.binaryLabel || 'the pinned binary'})` : 'detect() returned null');
+ if (!found.version) return fail('detect() returned no version');
+ return pass(`found ${found.version}`);
+ },
+ },
+ {
+ id: 'version-pin',
+ title: "the version matches the target's pin",
+ async run(target, env) {
+ const { found } = await env.detect();
+ if (!found?.version) return skip('no version to compare');
+ return String(found.version) === String(target.version)
+ ? pass(`pinned ${target.version}`)
+ : fail(`detected ${found.version}, pinned ${target.version}`);
+ },
+ },
+ {
+ id: 'fresh-turn',
+ title: 'one fresh turn answers PONG through the fake provider',
+ run: (target, env) => needsBinary(env, async () => {
+ const turn = await freshTurn(env);
+ if (!turn.ok) return fail(`turn failed: ${describeError(turn.error)}`);
+ if (!turn.requests.length) return fail('the provider was never called');
+ const text = String(turn.result.text ?? '');
+ return text.includes(PONG)
+ ? pass(`answered in ${turn.ms}ms with ${turn.requests.length} provider request(s)`)
+ : fail(`reply ${quote(text)} does not contain ${PONG}`);
+ }),
+ },
+ {
+ id: 'exact-reply',
+ title: 'the reply text is exact, with nothing extra',
+ run: (target, env) => needsBinary(env, async () => {
+ const turn = await freshTurn(env);
+ if (!turn.ok) return skip('no reply (fresh turn failed)');
+ const text = String(turn.result.text ?? '');
+ if (text === PONG) return pass(`reply is exactly ${PONG}`);
+ return text.trim() === PONG
+ ? pass(`reply is ${PONG} with surrounding whitespace ${quote(text)}`)
+ : fail(`reply ${quote(text)} is not exactly ${PONG}`);
+ }),
+ },
+ {
+ id: 'resume-session',
+ title: 'a resumed second turn keeps the session id',
+ run: (target, env) => needsBinary(env, async () => {
+ const first = await freshTurn(env);
+ if (!first.ok) return skip('no first turn to resume');
+ const id = first.result.newSessionId;
+ if (!id) return skip('the adapter returns no session id');
+ const second = await resumedTurn(env);
+ if (!second.ok) return fail(`resumed turn failed: ${describeError(second.error)}`);
+ const text = String(second.result.text ?? '');
+ if (!text.includes(RESUMED)) return fail(`resumed turn answered ${quote(text)}, expected ${RESUMED}`);
+ const next = second.result.newSessionId;
+ if (next && next !== id) return fail('resumed turn returned a different session id');
+ return pass(next ? 'session id kept across turns' : 'resumed; the adapter returned no new id, so the first is kept');
+ }),
+ },
+ {
+ id: 'refusal',
+ title: "a provider refusal surfaces as RunFailure kind 'refusal' (or 'refusal_retry_loop', stopped on the wire) and is not retried",
+ run: (target, env) => needsBinary(env, async () => {
+ const turn = await refusalTurn(env);
+ // Real CLIs make more than one provider call per turn (opencode also
+ // asks for a session title), so "retried" means more calls than an
+ // ordinary turn makes, measured on this target's own fresh turn.
+ const fresh = await freshTurn(env);
+ const baseline = fresh.ok && fresh.requests.length ? fresh.requests.length : 1;
+ const asked = turn.requests.filter((r) => r.directive === 'refuse').length;
+ if (turn.ok) return fail(`a refused turn reported success: ${quote(turn.result.text)}`);
+ if (!isNamedFailure(turn.error) || !REFUSAL_KINDS.has(turn.error.kind)) return fail(`expected RunFailure refusal or refusal_retry_loop, got ${describeError(turn.error)}`);
+ if (asked === 0) return fail('reported refusal without asking the provider');
+ if (asked > baseline) return fail(`the provider was asked ${asked} times against ${baseline} on an ordinary turn; a refusal must not be retried`);
+ return pass(`${turn.error.kind} named after ${asked} provider request(s), as on an ordinary turn`);
+ }),
+ },
+ {
+ id: 'provider-5xx',
+ title: 'a provider 5xx surfaces as a named failure, not a hang',
+ run: (target, env) => needsBinary(env, async () => {
+ const turn = await upstreamTurn(env);
+ if (turn.ok) return fail(`a 503 turn reported success: ${quote(turn.result.text)}`);
+ if (!isNamedFailure(turn.error)) return fail(describeError(turn.error));
+ // Ending only at the turn's own deadline means the 5xx was never
+ // surfaced: the CLI kept retrying until the adapter gave up on it.
+ if (['timeout', 'stuck'].includes(turn.error.kind)) return fail(`the 503 was not surfaced; the turn ran until RunFailure ${turn.error.kind} after ${turn.ms}ms`);
+ return pass(`RunFailure ${turn.error.kind} after ${turn.ms}ms`);
+ }),
+ },
+ {
+ id: 'retry-status',
+ title: 'while it retries a provider 5xx, the agent reports progress on its own stream',
+ timeoutMs: (env) => env.timeouts.retryWindowMs + env.timeouts.graceMs * 3,
+ run: (target, env) => needsProbe(env, async (turn, probe) => {
+ const window = `${turn.ms}ms`;
+ if (probe.failedRequests <= 1) return pass(`no retry: ${probe.failedRequests} provider request(s) in ${window}`);
+ const meta = probe.metaKeys.length ? `; _meta keys ${probe.metaKeys.join(', ')}` : '';
+ const answered = probe.answerMetaKeys.length ? `; the prompt answer carries _meta ${probe.answerMetaKeys.join(', ')}` : '';
+ const outside = Object.keys(probe.otherKinds).length ? `; outside the retries: ${listKinds(probe.otherKinds)}` : '';
+ if (!probe.updates) {
+ return fail(`${probe.failedRequests} provider requests in ${window} and no session/update between the first and the last${outside}${answered}`);
+ }
+ return pass(`${probe.updates} session/update(s) between the first and the last of ${probe.failedRequests} provider requests in ${window}: ${listKinds(probe.kinds)}${meta}${outside}${answered}; first ${probe.sample}`);
+ }),
+ },
+ {
+ id: 'retry-status-surfaced',
+ title: "the adapter's trace carries the agent's retry reports",
+ timeoutMs: (env) => env.timeouts.retryWindowMs + env.timeouts.graceMs * 3,
+ run: (target, env) => needsProbe(env, async (turn, probe) => {
+ if (!probe.updates) return skip('the agent sent no update during its retries, so there is nothing to surface');
+ const meta = probe.metaKeys.length ? ` (_meta keys ${probe.metaKeys.join(', ')})` : '';
+ if (probe.surfaced.length < probe.updates) {
+ return fail(`the agent sent ${listKinds(probe.kinds)}${meta}; the trace carries ${probe.surfaced.length ? probe.surfaced.join(', ') : 'no event'} from them`);
+ }
+ return pass(`${probe.updates} update(s) reached the trace as ${probe.surfaced.join(', ')}`);
+ }),
+ },
+ {
+ id: 'stall',
+ title: "a stalled provider ends within the adapter's timeout as a named failure",
+ timeoutMs: (env) => env.timeouts.stallMs + env.timeouts.graceMs * 2,
+ run: (target, env) => needsBinary(env, async () => {
+ const limit = env.timeouts.stallMs + env.timeouts.graceMs;
+ const turn = await stallTurn(env);
+ if (turn.ok) return fail(`a stalled turn reported success: ${quote(turn.result.text)}`);
+ if (!isNamedFailure(turn.error)) return fail(describeError(turn.error));
+ if (turn.error.kind === 'cancelled') return fail(`ended only when the runner cancelled it (${turn.ms}ms)`);
+ if (turn.ms > limit) return fail(`RunFailure ${turn.error.kind} after ${turn.ms}ms, over the ${limit}ms bound`);
+ return pass(`RunFailure ${turn.error.kind} after ${turn.ms}ms (timeout ${env.timeouts.stallMs}ms)`);
+ }),
+ },
+ {
+ id: 'key-not-in-argv',
+ title: 'the provider key never appears in argv',
+ timeoutMs: (env) => env.timeouts.turnMs + env.timeouts.stallMs + env.timeouts.graceMs * 2,
+ run: (target, env) => needsBinary(env, async () => {
+ const turns = (await Promise.all(ALL_TURNS.map((start) => start(env)))).filter(Boolean);
+ const processes = turns.reduce((n, t) => n + t.argv.processes, 0);
+ const leaks = turns.reduce((n, t) => n + t.argv.leaks, 0);
+ if (turns.some((t) => !t.argv.available)) return skip('ps is not available on this host');
+ if (leaks) return fail(`the key was in the argv of ${leaks} process(es)`);
+ if (!processes) return skip(`the adapter spawned no child process across ${turns.length} turns`);
+ return pass(`${processes} process(es) inspected across ${turns.length} turns`);
+ }),
+ },
+ {
+ id: 'key-not-in-output',
+ title: 'the provider key never appears in the reply or the error text',
+ timeoutMs: (env) => env.timeouts.turnMs + env.timeouts.stallMs + env.timeouts.graceMs * 2,
+ run: (target, env) => needsBinary(env, async () => {
+ const key = env.provider.apiKey;
+ const turns = (await Promise.all(ALL_TURNS.map((start) => start(env)))).filter(Boolean);
+ const where = [];
+ turns.forEach((t, index) => {
+ const surfaces = t.ok
+ ? [['reply', t.result.text], ['memory summary', t.result.memorySummary]]
+ : [['error', t.error?.message], ['error detail', t.error?.detail]];
+ surfaces.push(['trace', JSON.stringify(t.events)]);
+ for (const [name, text] of surfaces) {
+ if (text && String(text).includes(key)) where.push(`${name} of turn ${index + 1}`);
+ }
+ });
+ return where.length ? fail(`the key appeared in the ${where.join(', ')}`) : pass(`${turns.length} turns clean`);
+ }),
+ },
+ {
+ id: 'trace-order',
+ title: 'the trace has run events in order, when the adapter emits any',
+ run: (target, env) => needsBinary(env, async () => {
+ const turns = [['fresh', await freshTurn(env)], ['refusal', await refusalTurn(env)]];
+ const emitted = turns.reduce((n, [, t]) => n + t.events.length, 0);
+ if (!emitted) return skip('the adapter emits no trace events');
+ const problems = [];
+ let withRunEvents = 0;
+ for (const [name, t] of turns) {
+ const unknown = t.events.filter((e) => !e.known).map((e) => e.type);
+ if (unknown.length) problems.push(`${name}: unknown event types ${unknown.join(', ')}`);
+ const types = t.events.map((e) => e.type);
+ if (!types.some((type) => type.startsWith('run.'))) continue;
+ withRunEvents += 1;
+ const settled = types.filter((type) => type === 'run.settled').length;
+ if (types[0] !== 'run.started') problems.push(`${name}: first event is ${types[0]}, not run.started`);
+ if (settled !== 1) problems.push(`${name}: ${settled} run.settled events`);
+ if (types[types.length - 1] !== 'run.settled') problems.push(`${name}: run.settled is not the last event`);
+ const outcome = t.events.find((e) => e.type === 'run.settled')?.data?.outcome;
+ const expected = t.ok ? 'ok' : 'failed';
+ if (settled === 1 && outcome !== expected) problems.push(`${name}: settled ${outcome}, expected ${expected}`);
+ }
+ if (problems.length) return fail(problems.join('; '));
+ if (!withRunEvents) return skip(`${emitted} event(s), none of them run events`);
+ return pass(`${emitted} events, run events in order on ${withRunEvents} turn(s)`);
+ }),
+ },
+];
+
+/** Run one check with its own deadline. Never throws; never outlives the deadline. */
+export const runCheck = async (check, target, env, { timeoutMs } = {}) => {
+ const budget = timeoutMs ?? (typeof check.timeoutMs === 'function' ? check.timeoutMs(env) : check.timeoutMs)
+ ?? env.timeouts.checkMs;
+ const started = Date.now();
+ let timer;
+ const deadline = new Promise((resolve) => {
+ timer = setTimeout(() => resolve({ timedOut: true }), budget);
+ });
+ try {
+ const outcome = await Promise.race([
+ Promise.resolve().then(() => check.run(target, env)).then((value) => ({ value }), (error) => ({ error })),
+ deadline,
+ ]);
+ if (outcome.timedOut) {
+ env.cancelPending?.(`check ${check.id} ran out of time`);
+ return { status: 'fail', detail: `check timed out after ${budget}ms`, ms: Date.now() - started };
+ }
+ if (outcome.error) return { status: 'fail', detail: `check threw: ${describeError(outcome.error)}`, ms: Date.now() - started };
+ const { status, detail } = outcome.value || {};
+ if (!['pass', 'fail', 'skip'].includes(status)) return { status: 'fail', detail: `check returned status ${quote(status)}`, ms: Date.now() - started };
+ return { status, detail: String(detail ?? ''), ms: outcome.value.ms ?? Date.now() - started };
+ } finally {
+ clearTimeout(timer);
+ }
+};
diff --git a/packages/seats/compat/harness.js b/packages/seats/compat/harness.js
new file mode 100644
index 0000000000..23d53958be
--- /dev/null
+++ b/packages/seats/compat/harness.js
@@ -0,0 +1,267 @@
+/**
+ * The per-target harness the checks run against. It owns everything a check
+ * must not improvise: the fake provider, the run env, one memoized turn per
+ * prompt, the argv watch and the cleanup of anything an adapter left running.
+ *
+ * Checks see only the generic Adapter contract (src/contract.js) plus
+ * ctx.provider, so any adapter — stub, fake-cli, ACP — plugs in unchanged.
+ */
+import { execFile, spawn as childSpawn, spawnSync } from 'node:child_process';
+import { mkdtempSync, mkdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { delimiter, dirname, join } from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
+import { RunFailure, runFailure } from '../src/failures.js';
+import { isEventType } from '../src/trace/events.js';
+
+const PS_ARGS = ['-A', '-ww', '-o', 'pid=,ppid=,args='];
+const POLL_MS = 100;
+const SELF = new RegExp(`^(\\S*/)?ps ${PS_ARGS.join(' ')}\\s*$`);
+
+// One `ps` listing, reduced to this process's descendants. Argv is matched
+// against the key here and never kept, so a leak cannot travel further.
+// `ownPids` are the watch's own ps probes. Matching them by argv alone is not
+// enough: a probe that has exited but is not yet reaped lists as
+// "[ps] ", and one caught mid-exec lists as "[node]" or with this
+// runner's argv. Neither is the adapter's.
+export const descendantsFrom = (listing, rootPid, key, ownPids = new Set()) => {
+ const rows = [];
+ for (const line of String(listing || '').split('\n')) {
+ const match = /^\s*(\d+)\s+(\d+)\s(.*)$/.exec(line);
+ if (!match || SELF.test(match[3]) || ownPids.has(Number(match[1]))) continue;
+ rows.push({ pid: Number(match[1]), ppid: Number(match[2]), leaks: !!key && match[3].includes(key) });
+ }
+ const tree = new Set([rootPid]);
+ let grew = true;
+ while (grew) {
+ grew = false;
+ for (const row of rows) {
+ if (!tree.has(row.pid) && tree.has(row.ppid)) { tree.add(row.pid); grew = true; }
+ }
+ }
+ tree.delete(rootPid);
+ return rows.filter((row) => tree.has(row.pid));
+};
+
+/**
+ * Watch argv of every process this runner descends to while a turn is open.
+ * Sampled on the adapter's onSpawn (the child is alive at that instant) and on
+ * a short poll, which catches grandchildren and long-lived agent processes.
+ */
+export const watchArgv = (key, { rootPid = process.pid } = {}) => {
+ const seen = new Set();
+ const leaked = new Set();
+ // Every probe's pid, kept for the watch's life: a poll's listing can show a
+ // sync probe mid-exec and is only absorbed after spawnSync has returned it.
+ const probes = new Set();
+ let available = true;
+ let stopped = false;
+ const absorb = (listing) => {
+ for (const row of descendantsFrom(listing, rootPid, key, probes)) {
+ seen.add(row.pid);
+ if (row.leaks) leaked.add(row.pid);
+ }
+ };
+ const snapshot = () => {
+ const probe = spawnSync('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000 });
+ if (Number.isInteger(probe.pid)) probes.add(probe.pid);
+ if (probe.error || probe.status !== 0) { available = false; return; }
+ absorb(probe.stdout);
+ };
+ let inFlight = false;
+ const timer = setInterval(() => {
+ if (inFlight) return;
+ inFlight = true;
+ const child = execFile('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000, maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => {
+ inFlight = false;
+ // A poll that lands after stop() would change a report already read.
+ if (stopped) return;
+ if (error) available = false; else absorb(stdout);
+ });
+ if (Number.isInteger(child.pid)) probes.add(child.pid);
+ }, POLL_MS);
+ timer.unref();
+ return {
+ snapshot,
+ stop() { clearInterval(timer); snapshot(); stopped = true; },
+ get report() { return { available, processes: seen.size, leaks: leaked.size }; },
+ };
+};
+
+/** Kill whatever an adapter left behind under this runner (best effort). */
+export const killDescendants = (rootPid = process.pid) => {
+ const probe = spawnSync('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000 });
+ if (probe.error || probe.status !== 0) return 0;
+ const rows = descendantsFrom(probe.stdout, rootPid, '');
+ for (const row of rows) {
+ try { process.kill(row.pid, 'SIGKILL'); } catch { /* already gone */ }
+ }
+ return rows.length;
+};
+
+const WIRE_LIMIT = 2000;
+
+/**
+ * A spawn that also records the child's newline-delimited JSON on stdin and
+ * stdout, so a probe sees what the agent said before any adapter read it.
+ * Lines that are not JSON are not protocol messages and are not kept. The
+ * listener is attached in the spawning tick, before the adapter's own, so no
+ * chunk is lost to either reader. `stamp()` adds fields to each record; the
+ * harness stamps how many provider requests the turn had made by then.
+ */
+export const tapSpawn = (wire, spawnImpl = childSpawn, { stamp = () => ({}) } = {}) => (command, args, options) => {
+ const child = spawnImpl(command, args, options);
+ const started = Date.now();
+ const reader = (dir) => {
+ const decoder = new StringDecoder('utf8');
+ let buffer = '';
+ return (chunk) => {
+ buffer += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+ let at;
+ while ((at = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, at).trim();
+ buffer = buffer.slice(at + 1);
+ if (!line || wire.length >= WIRE_LIMIT) continue;
+ try { wire.push({ dir, ms: Date.now() - started, ...stamp(), msg: JSON.parse(line) }); } catch { /* not a protocol line */ }
+ }
+ };
+ };
+ child.stdout?.on('data', reader('in'));
+ if (child.stdin) {
+ const write = child.stdin.write.bind(child.stdin);
+ const out = reader('out');
+ child.stdin.write = (chunk, ...rest) => { out(chunk); return write(chunk, ...rest); };
+ }
+ return child;
+};
+
+// Only names a CLI needs to start: never the host's own secrets.
+const runEnv = ({ home, binaryPath }) => {
+ const xdg = (name) => { const dir = join(home, name); mkdirSync(dir, { recursive: true }); return dir; };
+ const path = [binaryPath ? dirname(binaryPath) : null, process.env.PATH].filter(Boolean).join(delimiter);
+ return {
+ PATH: path,
+ HOME: home,
+ TMPDIR: xdg('tmp'),
+ XDG_CONFIG_HOME: xdg('.config'),
+ XDG_DATA_HOME: xdg('.local/share'),
+ XDG_STATE_HOME: xdg('.local/state'),
+ XDG_CACHE_HOME: xdg('.cache'),
+ LANG: process.env.LANG || 'C.UTF-8',
+ NO_COLOR: '1',
+ };
+};
+
+/**
+ * @param {object} opts
+ * @param {import('../src/contract.js').Adapter} opts.adapter
+ * @param {{ baseURL: string, apiKey: string, model: string, requests: object[] }} opts.provider
+ * @param {string} [opts.binaryPath]
+ * @param {{ turnMs: number, stallMs: number, retryWindowMs?: number }} opts.timeouts
+ * @param {import('../src/contract.js').Adapter} [opts.probeAdapter] the same adapter with
+ * the target's `probeOptions` applied, for checks that read the agent itself
+ */
+export const createHarness = ({ adapter, provider, binaryPath = null, timeouts, probeAdapter = null }) => {
+ const root = mkdtempSync(join(tmpdir(), 'seats-compat-'));
+ const home = join(root, 'home');
+ const cwd = join(root, 'work');
+ mkdirSync(home, { recursive: true });
+ mkdirSync(cwd, { recursive: true });
+ const env = runEnv({ home, binaryPath });
+ const pending = new Set();
+ const turns = new Map();
+ let detection;
+
+ const detect = () => {
+ if (!detection) {
+ detection = Promise.resolve().then(() => adapter.detect({ env, binaryPath }))
+ .then((found) => ({ found: found || null }), (error) => ({ found: null, error }));
+ }
+ return detection;
+ };
+
+ const runTurn = async (prompt, { sessionId, timeoutMs = timeouts.turnMs, via = adapter, tap = false } = {}) => {
+ const events = [];
+ const wire = [];
+ const tapped = { spawned: false };
+ const firstRequest = provider.requests.length;
+ // How many provider requests this turn had made at a given instant; an
+ // event and a wire line stamped with it can be placed among the retries.
+ const stamp = () => ({ requests: provider.requests.length - firstRequest });
+ const controller = new AbortController();
+ pending.add(controller);
+ const watch = watchArgv(provider.apiKey);
+ const trace = {
+ emit(type, data = {}) { events.push({ seq: events.length + 1, type, data, known: isEventType(type), ...stamp() }); },
+ };
+ const ctx = {
+ provider: { baseURL: provider.baseURL, apiKey: provider.apiKey, model: provider.model },
+ env: { ...env },
+ cwd,
+ trace,
+ signal: controller.signal,
+ onSpawn: () => watch.snapshot(),
+ timeoutMs,
+ agentName: 'compat',
+ ...(sessionId ? { sessionId } : {}),
+ // ACP adapters take ctx._spawnImpl; one that ignores it leaves the
+ // wire empty and spawned stays false, which the probe checks report.
+ ...(tap ? { _spawnImpl: (...args) => { tapped.spawned = true; return tapSpawn(wire, childSpawn, { stamp })(...args); } } : {}),
+ };
+ const started = Date.now();
+ let outcome;
+ try {
+ const result = await via.spawn(prompt, ctx);
+ outcome = { ok: true, result: result || {} };
+ } catch (error) {
+ outcome = { ok: false, error };
+ } finally {
+ watch.stop();
+ pending.delete(controller);
+ }
+ return {
+ ...outcome,
+ ms: Date.now() - started,
+ events,
+ argv: watch.report,
+ requests: provider.requests.slice(firstRequest),
+ ...(tap ? { wire, tapped: tapped.spawned } : {}),
+ };
+ };
+
+ /** The same named turn is run once per target and shared by every check that reads it. */
+ const turn = (name, prompt, opts) => {
+ if (!turns.has(name)) turns.set(name, runTurn(prompt, opts));
+ return turns.get(name);
+ };
+
+ /**
+ * A named turn on the probe adapter with the agent's stdio recorded, or
+ * null when the target declares no probe. Memoized like turn().
+ */
+ const probeTurn = (name, prompt, opts = {}) => {
+ if (!probeAdapter) return null;
+ const key = `probe:${name}`;
+ if (!turns.has(key)) turns.set(key, runTurn(prompt, { ...opts, via: probeAdapter, tap: true }));
+ return turns.get(key);
+ };
+
+ // A check that ran out of time leaves its turn open; cancel it so the next
+ // check starts clean and the adapter is told to stop its child.
+ const cancelPending = (why) => {
+ for (const controller of pending) controller.abort(runFailure('cancelled', why));
+ };
+
+ const dispose = () => {
+ cancelPending('the compatibility run finished');
+ rmSync(root, { recursive: true, force: true });
+ };
+
+ return {
+ adapter, probeAdapter, provider, binaryPath, timeouts, env, cwd,
+ detect, turn, probeTurn, cancelPending, dispose,
+ };
+};
+
+export const isNamedFailure = (error) => error instanceof RunFailure
+ || (error?.name === 'RunFailure' && typeof error?.kind === 'string');
diff --git a/packages/seats/compat/install.js b/packages/seats/compat/install.js
new file mode 100644
index 0000000000..ee63a8819d
--- /dev/null
+++ b/packages/seats/compat/install.js
@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * Install each enabled target's pinned binary into a job-local prefix.
+ *
+ * node compat/install.js --targets compat/targets.json --prefix
+ *
+ * Nothing is installed globally. A release asset is downloaded, checked
+ * against its pinned sha256 BEFORE it is unpacked, and extracted into
+ * /-/. An npm spec is installed into that directory with
+ * --ignore-scripts, so no package's install hook runs. A pip spec is installed
+ * into /venv from wheels only (--only-binary=:all:), so no package's
+ * setup.py runs either; the interpreter is python3, or COMPAT_PYTHON. A target whose adapter
+ * module is absent is not installed: it would only be skipped by the runner.
+ */
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { installDirOf, parseAdapterSpec, platformKey, resolveBinary } from './targets.js';
+import { loadTargets } from './run.js';
+
+const sha256 = (buffer) => createHash('sha256').update(buffer).digest('hex');
+
+const run = (command, args, opts = {}) => {
+ const result = spawnSync(command, args, { stdio: ['ignore', 'inherit', 'inherit'], ...opts });
+ if (result.error) throw result.error;
+ if (result.status !== 0) throw new Error(`${command} ${args[0] ?? ''} exited ${result.status}`);
+};
+
+export const installRelease = async ({ url, sha256: expected }, dir, { fetchImpl = fetch } = {}) => {
+ if (!/^[0-9a-f]{64}$/.test(String(expected || ''))) throw new Error(`release asset ${url} has no pinned sha256`);
+ const response = await fetchImpl(url, { redirect: 'follow' });
+ if (!response.ok) throw new Error(`download ${url} answered ${response.status}`);
+ const body = Buffer.from(await response.arrayBuffer());
+ const actual = sha256(body);
+ if (actual !== expected) throw new Error(`sha256 mismatch for ${url}: expected ${expected}, got ${actual}`);
+ mkdirSync(resolve(dir, '..'), { recursive: true });
+ const scratch = mkdtempSync(join(resolve(dir, '..'), '.download-'));
+ try {
+ const archive = join(scratch, 'asset');
+ writeFileSync(archive, body);
+ mkdirSync(dir, { recursive: true });
+ const flag = /\.tar\.bz2$|\.tbz2?$/.test(url) ? '-xjf' : '-xzf';
+ run('tar', [flag, archive, '-C', dir]);
+ } finally {
+ rmSync(scratch, { recursive: true, force: true });
+ }
+};
+
+export const installNpm = (spec, dir) => {
+ mkdirSync(dir, { recursive: true });
+ run('npm', ['install', '--prefix', dir, '--ignore-scripts', '--no-save', '--no-audit', '--no-fund', spec]);
+};
+
+/** A pinned `name==version` spec, installed from wheels into /venv. */
+export const installPip = (spec, dir, { python = process.env.COMPAT_PYTHON || 'python3', runImpl = run } = {}) => {
+ if (!/^[A-Za-z0-9][A-Za-z0-9._-]*==[A-Za-z0-9][A-Za-z0-9.+_-]*$/.test(String(spec || ''))) {
+ throw new Error(`pip spec ${JSON.stringify(spec)} must pin one exact version (name==version)`);
+ }
+ mkdirSync(dir, { recursive: true });
+ const venv = join(dir, 'venv');
+ runImpl(python, ['-m', 'venv', venv]);
+ runImpl(join(venv, 'bin', 'pip'), [
+ 'install', '--no-input', '--disable-pip-version-check', '--no-cache-dir', '--only-binary=:all:', spec,
+ ]);
+};
+
+export const installTarget = async (target, { prefix, platform = platformKey(), fetchImpl, pipOptions } = {}) => {
+ const binary = target.binary;
+ if (target.enabled === false) return { skipped: 'disabled' };
+ if (!binary || binary.path) return { skipped: 'no binary to install' };
+ if (!existsSync(parseAdapterSpec(target.adapter).path)) return { skipped: `adapter module ${target.adapter} is not present` };
+ const resolved = resolveBinary(target, { prefix, platform });
+ if (resolved?.missing) return { skipped: resolved.missing };
+ const dir = installDirOf(target, prefix);
+ if (binary.release) await installRelease(binary.release[platform], dir, { fetchImpl });
+ else if (binary.npm) installNpm(binary.npm, dir);
+ else if (binary.pip) installPip(binary.pip, dir, pipOptions);
+ else return { skipped: 'binary names neither release, npm nor pip' };
+ if (!existsSync(resolved.path)) throw new Error(`${target.cli} installed but ${resolved.label} is missing`);
+ chmodSync(resolved.path, 0o755);
+ return { installed: resolved.path };
+};
+
+const argValue = (argv, name) => {
+ const at = argv.indexOf(name);
+ return at === -1 ? null : argv[at + 1] ?? null;
+};
+
+export const main = async (argv = process.argv.slice(2)) => {
+ const targetsPath = argValue(argv, '--targets');
+ const prefix = argValue(argv, '--prefix');
+ if (!targetsPath || !prefix) {
+ process.stderr.write('usage: node compat/install.js --targets --prefix \n');
+ return 2;
+ }
+ let failed = 0;
+ for (const target of loadTargets(resolve(targetsPath))) {
+ try {
+ const outcome = await installTarget(target, { prefix: resolve(prefix) });
+ process.stdout.write(`${target.cli}@${target.version}: ${outcome.installed ? `installed ${outcome.installed}` : `skipped (${outcome.skipped})`}\n`);
+ } catch (error) {
+ // One CLI failing to install must not hide the rest of the matrix; the
+ // runner reports it as "binary not installed".
+ failed += 1;
+ process.stderr.write(`${target.cli}@${target.version}: install failed: ${error?.message || error}\n`);
+ }
+ }
+ return failed ? 1 : 0;
+};
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ main().then((code) => process.exit(code), (error) => {
+ process.stderr.write(`compat install failed: ${error?.stack || error}\n`);
+ process.exit(1);
+ });
+}
diff --git a/packages/seats/compat/render.js b/packages/seats/compat/render.js
new file mode 100644
index 0000000000..dd7f74445d
--- /dev/null
+++ b/packages/seats/compat/render.js
@@ -0,0 +1,171 @@
+/**
+ * Render a results document as one self-contained HTML page: no external
+ * scripts, styles, fonts or images, so the artifact opens anywhere and a
+ * Pages deploy needs nothing but this file. Every value from the results is
+ * escaped — detail strings carry CLI and provider output.
+ */
+const ESCAPES = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' };
+export const escapeHtml = (value) => String(value ?? '').replace(/[&<>"']/g, (c) => ESCAPES[c]);
+
+const MARK = { pass: '✓', fail: '✗', skip: '–', limit: '!' };
+const STATUSES = new Set(['pass', 'fail', 'skip']);
+
+/** A failing row is a limit when its target declares one for that check (run.js limitOf). */
+export const isLimitRow = (row) => row?.status === 'fail' && typeof row.limit?.note === 'string' && !!row.limit.note;
+const statusOf = (row) => {
+ if (isLimitRow(row)) return 'limit';
+ return row && STATUSES.has(row.status) ? row.status : 'missing';
+};
+
+/** "limit of opencode@1.19.4: . Fixed in ." as plain text. */
+export const limitText = (row) => {
+ const { note, fixedIn } = row?.limit || {};
+ if (!note) return '';
+ const where = row.target ? ` of ${row.target}` : ' of this version';
+ return `limit${where}: ${note}${fixedIn ? `. Fixed in ${fixedIn}` : ''}`;
+};
+
+const STYLE = `
+:root{--bg:#fbfbfa;--fg:#1c1c1a;--muted:#6b6b66;--line:#e3e2de;--card:#fff;
+--pass:#1f7a3d;--pass-bg:#e6f4ea;--fail:#b3261e;--fail-bg:#fbe9e7;--skip:#6b6b66;--skip-bg:#efeeea;
+--limit:#8a5300;--limit-bg:#fdf0d5;}
+@media (prefers-color-scheme:dark){:root{--bg:#161615;--fg:#ecebe8;--muted:#a3a29c;--line:#34332f;--card:#1f1e1c;
+--pass:#7fd79a;--pass-bg:#173523;--fail:#ff9a8f;--fail-bg:#3d1c19;--skip:#a3a29c;--skip-bg:#2a2926;
+--limit:#f2c46b;--limit-bg:#3a2c10;}}
+*{box-sizing:border-box}
+body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",system-ui,sans-serif}
+main{max-width:1120px;margin:0 auto;padding:24px 16px 48px}
+h1{font-size:22px;margin:0 0 4px}
+.meta{color:var(--muted);font-size:13px;margin:0 0 16px;overflow-wrap:anywhere}
+.totals{display:flex;gap:8px;flex-wrap:wrap;margin:0 0 16px;padding:0;list-style:none}
+.totals li{padding:2px 10px;border-radius:999px;font-size:13px;font-weight:600}
+.scroll{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--card)}
+table{border-collapse:collapse;width:100%;font-size:14px}
+th,td{padding:8px 10px;border-bottom:1px solid var(--line);text-align:center;vertical-align:top}
+thead th{font-size:12px;color:var(--muted);font-weight:600;white-space:nowrap}
+th.t,td.t{text-align:left;position:sticky;left:0;background:var(--card);white-space:nowrap;z-index:1}
+td.t small{display:block;color:var(--muted);font-weight:400;white-space:normal;max-width:220px}
+.pass{color:var(--pass);background:var(--pass-bg)}.fail{color:var(--fail);background:var(--fail-bg)}
+.skip,.missing{color:var(--skip);background:var(--skip-bg)}.limit{color:var(--limit);background:var(--limit-bg)}
+td.c{font-weight:700;min-width:36px;position:relative}
+tr.d td{text-align:left;background:var(--bg);font-size:13px}
+tr.d ul{margin:0;padding-left:18px}
+/* Details stay readable without scrolling the grid sideways on a phone. */
+.dd{position:sticky;left:10px;max-width:min(100%,calc(100vw - 54px))}tr.d li{overflow-wrap:anywhere}
+tr.d code,details code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px}
+details{margin-top:4px}summary{cursor:pointer;color:var(--muted)}
+/* .sr stays inside its positioned cell; unanchored, it widened the page at 390px. */
+.sr{position:absolute;left:0;top:0;width:1px;height:1px;overflow:hidden;clip-path:inset(50%);white-space:nowrap}
+ol.legend{padding-left:22px;color:var(--muted);font-size:13px}
+.up{border:1px solid var(--line);border-radius:10px;background:var(--card);padding:12px 14px;margin:0 0 10px;overflow-wrap:anywhere}
+.up h3{font-size:15px;margin:0 0 6px}.up p,.up ul{margin:4px 0;font-size:14px}.up ul{padding-left:18px}
+.badge{display:inline-block;padding:1px 8px;border-radius:999px;font-size:12px;font-weight:600;margin-right:6px}
+.worked,.fixed{color:var(--pass);background:var(--pass-bg)}.open{color:var(--fail);background:var(--fail-bg)}
+.up code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px}
+@media (max-width:480px){main{padding:16px}th,td{padding:6px}td.t small{max-width:140px}}
+`;
+
+const cell = (row, check) => {
+ const status = statusOf(row);
+ const limit = status === 'limit' ? ` (${limitText(row)})` : '';
+ const label = `${check.title}: ${status}${limit}${row?.detail ? ` — ${row.detail}` : ''}`;
+ return `${MARK[status] || '?'}${escapeHtml(status)} | `;
+};
+
+// A limit leads with what the version cannot do and where it is fixed; a
+// declared limit on a passing row says it did not reproduce this run.
+const limitNote = (row) => {
+ if (!row?.limit?.note) return '';
+ if (isLimitRow(row)) return ` ${escapeHtml(limitText(row))}`;
+ if (row.status === 'pass') return ` (declared ${escapeHtml(limitText(row))}; it did not reproduce)`;
+ return '';
+};
+
+const detailItem = (check, row) => `${escapeHtml(check.id)} ${escapeHtml(statusOf(row))}${limitNote(row)}${row?.detail ? `: ${escapeHtml(row.detail)}` : ''}${Number.isFinite(row?.ms) && row.ms ? ` (${escapeHtml(row.ms)}ms)` : ''}`;
+
+const targetRows = (target, checks, rows) => {
+ const byCheck = new Map(rows.filter((r) => r.target === target.target).map((r) => [r.check, r]));
+ const failing = checks.filter((c) => ['fail', 'limit'].includes(statusOf(byCheck.get(c.id))) || !byCheck.has(c.id));
+ const note = target.note || (target.requires ? `requires ${target.requires}` : '');
+ const head = `| ${escapeHtml(target.target)}${note ? `${escapeHtml(note)}` : ''} | ${checks.map((c) => cell(byCheck.get(c.id), c)).join('')}
`;
+ const all = `All ${checks.length} results
${checks.map((c) => detailItem(c, byCheck.get(c.id))).join('')}
`;
+ const failures = failing.length ? `${failing.map((c) => detailItem(c, byCheck.get(c.id))).join('')}
` : '';
+ return `${head}${failures}${all} |
`;
+};
+
+// Only an https link becomes an anchor; anything else stays text.
+const reference = (url) => (/^https:\/\/[^\s"<>]+$/.test(url)
+ ? `${escapeHtml(url)}` : escapeHtml(url));
+
+/**
+ * One upstream bug: its status, the workaround (or the planned one) and where
+ * it lives, the tests that prove it, what this run's matrix saw, and the
+ * upstream links as references only.
+ */
+const BADGE = { 'worked around': 'worked', 'fixed upstream': 'fixed' };
+
+export const upstreamItem = (row) => {
+ const worked = row?.status === 'worked around';
+ const fixed = row?.status === 'fixed upstream';
+ const w = row?.workaround || {};
+ const where = [w.ref && `${escapeHtml(w.ref)}`, w.where && escapeHtml(w.where)].filter(Boolean).join(' · ');
+ const tests = Array.isArray(w.tests) && w.tests.length
+ ? `${w.tests.map((t) => `${escapeHtml(t)} `).join('')}
`
+ : (fixed ? '' : 'No test proves it yet.
');
+ const fixedLine = fixed ? `Fixed upstream in ${escapeHtml(row.fixed?.version)}: ${escapeHtml(row.fixed?.verified)}
\n` : '';
+ const label = worked ? 'Workaround' : fixed ? 'Our workaround, for older versions' : 'Planned workaround';
+ const matrix = row?.matrix ? `This run: ${escapeHtml(row.matrix.target)} · ${escapeHtml(row.matrix.check)} ${escapeHtml(row.matrix.observed || 'not run')}
` : '';
+ const refs = Array.isArray(row?.references) && row.references.length
+ ? `Upstream (reference, not the fix): ${row.references.map(reference).join(' · ')}
` : '';
+ return `${BADGE[row?.status] ? escapeHtml(row.status) : 'open'}${escapeHtml(row?.cli)}${row?.versions ? ` ${escapeHtml(row.versions)}` : ''}
+${escapeHtml(row?.bug)}
+${row?.downgraded ? `${escapeHtml(row.downgraded)}
` : ''}${fixedLine}${label}: ${escapeHtml(w.name)}${where ? ` (${where})` : ''}
+${tests}${matrix}${refs}`;
+};
+
+const upstreamSection = (upstream) => {
+ if (!Array.isArray(upstream) || !upstream.length) return '';
+ const worked = upstream.filter((r) => r.status === 'worked around').length;
+ const fixed = upstream.filter((r) => r.status === 'fixed upstream').length;
+ return `Upstream bugs
+${worked} worked around · ${fixed} fixed upstream · ${upstream.length - worked - fixed} open. A row is worked around only when its workaround is on a pushed commit and a test there proves it, and fixed upstream only when we checked the fixing release ourselves.
+${upstream.map(upstreamItem).join('\n')}`;
+};
+
+/** @param {object} results the document run.js writes as results.json */
+export const renderMatrix = (results) => {
+ const checks = Array.isArray(results?.checks) ? results.checks : [];
+ const rows = Array.isArray(results?.rows) ? results.rows : [];
+ const targets = Array.isArray(results?.targets) ? results.targets : [];
+ const host = results?.host || {};
+ const count = (s) => rows.filter((r) => statusOf(r) === s).length;
+ return `
+
+
+
+
+Seats compatibility
+
+
+
+
+Seats compatibility
+Generated ${escapeHtml(results?.generatedAt)} on ${escapeHtml(host.os)} ${escapeHtml(host.arch)}, node ${escapeHtml(host.node)} · schema ${escapeHtml(results?.schema)}
+- ${count('pass')} pass
- ${count('fail')} fail
- ${count('limit')} limit
- ${count('skip')} skip
+
+${upstreamSection(results?.upstream)}
+Checks
+${MARK.pass} pass · ${MARK.fail} fail · ${MARK.limit} limit of that CLI version (a failure its target declares, with where it is fixed) · ${MARK.skip} skip
+${checks.map((c) => `${escapeHtml(c.id)} ${escapeHtml(c.title)} `).join('')}
+
+
+
+`;
+};
diff --git a/packages/seats/compat/run.js b/packages/seats/compat/run.js
new file mode 100644
index 0000000000..e270807043
--- /dev/null
+++ b/packages/seats/compat/run.js
@@ -0,0 +1,199 @@
+#!/usr/bin/env node
+/**
+ * The compatibility matrix runner.
+ *
+ * node compat/run.js --targets compat/targets.json --out [--prefix ]
+ *
+ * Starts the fake provider, runs every enabled target against every check,
+ * and writes /results.json and /index.html. A failing check is data,
+ * not a crash: the exit code is 0 whenever the matrix was produced.
+ * `--prefix` (or COMPAT_PREFIX) is where compat/install.js put the binaries.
+ */
+import { randomBytes } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { arch, platform, release } from 'node:os';
+import { delimiter, dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { startFakeProvider } from '../test-support/fake-provider.js';
+import { CHECKS, runCheck } from './checks.js';
+import { createHarness, killDescendants } from './harness.js';
+import { isLimitRow, renderMatrix } from './render.js';
+import { loadAdapter, resolveBinary, targetLabel } from './targets.js';
+import { loadUpstream, upstreamForResults } from './upstream.js';
+
+export { isLimitRow };
+export const RESULTS_SCHEMA = 'seats-compat/1';
+export const DEFAULT_TIMEOUTS = Object.freeze({
+ turnMs: 30000, stallMs: 8000, graceMs: 5000, checkMs: 45000, retryWindowMs: 20000,
+});
+
+/**
+ * A target may declare `limits: { : { note, fixedIn? } }`: a check
+ * that fails for a reason that belongs to that CLI version, not to the
+ * adapter. The row keeps its real status; the limit rides along so the page
+ * can say "limit of this version" and where it is fixed, and so a declared
+ * limit that stops reproducing is visible as a pass that carries one.
+ */
+export const limitOf = (target, checkId) => {
+ const limit = target?.limits?.[checkId];
+ if (!limit || typeof limit.note !== 'string' || !limit.note) return null;
+ return { note: limit.note, ...(typeof limit.fixedIn === 'string' && limit.fixedIn ? { fixedIn: limit.fixedIn } : {}) };
+};
+
+/** pass / fail / skip / limit counts; `fail` counts only failures no limit explains. */
+export const totalsOf = (rows) => ({
+ pass: rows.filter((r) => r.status === 'pass').length,
+ fail: rows.filter((r) => r.status === 'fail' && !isLimitRow(r)).length,
+ skip: rows.filter((r) => r.status === 'skip').length,
+ limit: rows.filter(isLimitRow).length,
+});
+
+const rowsFor = (target, statusOf) => CHECKS.map((check) => {
+ const { status, detail } = statusOf(check);
+ return { target: targetLabel(target), cli: target.cli, version: target.version, check: check.id, status, detail, ms: 0 };
+});
+
+const timeoutsFor = (target, overrides) => {
+ const merged = { ...DEFAULT_TIMEOUTS, ...(target.timeouts || {}), ...(overrides || {}) };
+ if (!overrides?.checkMs && !target.timeouts?.checkMs) merged.checkMs = merged.turnMs + 15000;
+ return merged;
+};
+
+// Adapters whose detect() reads the host PATH see the pinned binary first,
+// for the length of this target only.
+const withPath = async (binaryPath, fn) => {
+ if (!binaryPath) return fn();
+ const before = process.env.PATH;
+ process.env.PATH = [dirname(binaryPath), before].filter(Boolean).join(delimiter);
+ try { return await fn(); } finally { process.env.PATH = before; }
+};
+
+/** Run one target through the suite. Always returns one row per check. */
+export const runTarget = async (target, { provider, prefix, checks = CHECKS, timeouts, log = () => {} } = {}) => {
+ const label = targetLabel(target);
+ const all = (status, detail) => rowsFor(target, () => ({ status, detail })).filter((row) => checks.some((c) => c.id === row.check));
+ if (target.enabled === false) return all('skip', 'target disabled');
+ const binary = resolveBinary(target, { prefix });
+ if (binary?.missing) return all('skip', binary.missing);
+ const loaded = await loadAdapter(target, { binaryPath: binary?.path });
+ if (loaded.skip) return all('skip', loaded.skip);
+ if (loaded.error) return all('fail', loaded.error);
+ if (binary?.path && !existsSync(binary.path)) {
+ return checks.map((check) => ({
+ target: label, cli: target.cli, version: target.version, check: check.id, ms: 0,
+ ...(check.id === 'detect' ? { status: 'fail', detail: `binary not installed at ${binary.label}` } : { status: 'skip', detail: 'binary not installed' }),
+ }));
+ }
+ // The probe instance is the same export built with probeOptions on top, so
+ // a check can see the agent with nothing of the adapter's own in between.
+ let probeAdapter = null;
+ if (target.probeOptions) {
+ const probe = await loadAdapter({ ...target, options: { ...(target.options || {}), ...target.probeOptions } }, { binaryPath: binary?.path });
+ if (probe.adapter && probe.adapter !== loaded.adapter) probeAdapter = probe.adapter;
+ else log(`${label}: probeOptions ignored (${probe.error || probe.skip || 'the export is an adapter, not a factory'})`);
+ }
+ const harness = createHarness({
+ adapter: loaded.adapter, provider, binaryPath: binary?.path, timeouts: timeoutsFor(target, timeouts), probeAdapter,
+ });
+ const rows = [];
+ try {
+ await withPath(binary?.path, async () => {
+ for (const check of checks) {
+ const outcome = await runCheck(check, { ...target, binaryLabel: binary?.label }, harness);
+ log(`${label} ${check.id}: ${outcome.status}${outcome.detail ? ` — ${outcome.detail}` : ''}`);
+ const limit = limitOf(target, check.id);
+ rows.push({ target: label, cli: target.cli, version: target.version, check: check.id, ...outcome, ...(limit ? { limit } : {}) });
+ }
+ });
+ } finally {
+ harness.dispose();
+ killDescendants();
+ }
+ return rows;
+};
+
+export const hostInfo = () => ({ os: `${platform()} ${release()}`, platform: platform(), arch: arch(), node: process.version });
+
+/** Run every target and return the results document (not yet written). */
+export const runMatrix = async (targets, { prefix, checks = CHECKS, timeouts, log, upstream = [], now = () => new Date() } = {}) => {
+ const fake = await startFakeProvider();
+ // A key-shaped value that is fake by construction: the leak checks look for
+ // exactly this string, and no real provider would accept it.
+ const provider = { ...fake, apiKey: `sk-compat-${randomBytes(16).toString('hex')}` };
+ const rows = [];
+ try {
+ for (const target of targets) rows.push(...await runTarget(target, { provider, prefix, checks, timeouts, log }));
+ } finally {
+ await fake.close();
+ }
+ return {
+ schema: RESULTS_SCHEMA,
+ generatedAt: now().toISOString(),
+ host: hostInfo(),
+ checks: checks.map(({ id, title }) => ({ id, title })),
+ targets: targets.map((t) => ({
+ target: targetLabel(t), cli: t.cli, version: t.version, adapter: t.adapter,
+ enabled: t.enabled !== false, requires: t.requires || null, note: t.note || null,
+ })),
+ rows,
+ totals: totalsOf(rows),
+ upstream: upstreamForResults(upstream, rows),
+ };
+};
+
+export const writeOutputs = (results, outDir) => {
+ mkdirSync(outDir, { recursive: true });
+ const json = join(outDir, 'results.json');
+ const html = join(outDir, 'index.html');
+ writeFileSync(json, `${JSON.stringify(results, null, 2)}\n`);
+ writeFileSync(html, renderMatrix(results));
+ return { json, html };
+};
+
+export const loadTargets = (path) => {
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
+ const targets = Array.isArray(parsed) ? parsed : parsed?.targets;
+ if (!Array.isArray(targets)) throw new Error(`${path} must hold an array of targets`);
+ for (const t of targets) {
+ if (!t?.cli || !t?.version || !t?.adapter) throw new Error(`every target needs cli, version and adapter: ${JSON.stringify(t)}`);
+ }
+ return targets;
+};
+
+const argValue = (argv, name) => {
+ const at = argv.indexOf(name);
+ return at === -1 ? null : argv[at + 1] ?? null;
+};
+
+export const main = async (argv = process.argv.slice(2)) => {
+ const targetsPath = argValue(argv, '--targets');
+ const out = argValue(argv, '--out');
+ if (!targetsPath || !out) {
+ process.stderr.write('usage: node compat/run.js --targets --out [--prefix ] [--upstream ]\n');
+ return 2;
+ }
+ const prefix = argValue(argv, '--prefix') || process.env.COMPAT_PREFIX || null;
+ // Upstream bug rows live beside the targets unless --upstream names a file.
+ const named = argValue(argv, '--upstream');
+ const upstreamPath = resolve(named || join(dirname(resolve(targetsPath)), 'upstream.json'));
+ if (named && !existsSync(upstreamPath)) {
+ process.stderr.write(`no upstream file at ${upstreamPath}\n`);
+ return 2;
+ }
+ const results = await runMatrix(loadTargets(resolve(targetsPath)), {
+ prefix, upstream: loadUpstream(upstreamPath), log: (line) => process.stdout.write(`${line}\n`),
+ });
+ const paths = writeOutputs(results, resolve(out));
+ const { pass, fail, skip, limit } = results.totals;
+ process.stdout.write(`pass ${pass} fail ${fail} limit ${limit} skip ${skip}\n${paths.json}\n${paths.html}\n`);
+ return 0;
+};
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ // Exit explicitly: an adapter that ignored cancellation must not keep the
+ // nightly job alive after the matrix is written.
+ main().then((code) => process.exit(code), (error) => {
+ process.stderr.write(`compat run failed: ${error?.stack || error}\n`);
+ process.exit(1);
+ });
+}
diff --git a/packages/seats/compat/targets.js b/packages/seats/compat/targets.js
new file mode 100644
index 0000000000..b8cf102a0c
--- /dev/null
+++ b/packages/seats/compat/targets.js
@@ -0,0 +1,77 @@
+/**
+ * Targets are data: `{ cli, version, adapter: 'module#export', binary?, enabled,
+ * requires?, options?, timeouts?, note? }`. This module turns one entry into
+ * an adapter and a binary path, or into the reason it cannot run.
+ *
+ * Module specs resolve against the package root, so `src/adapters/stub.js`
+ * means packages/seats/src/adapters/stub.js wherever the runner is started.
+ * The export may be an adapter object or a factory taking `options`.
+ */
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { isAbsolute, join, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+export const PACKAGE_ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)));
+export const platformKey = (platform = process.platform, arch = process.arch) => `${platform}-${arch}`;
+
+export const parseAdapterSpec = (spec) => {
+ const [file, name = 'default'] = String(spec || '').split('#');
+ if (!file) throw new Error(`adapter spec ${JSON.stringify(spec)} names no module`);
+ return { file, exportName: name, path: isAbsolute(file) ? file : join(PACKAGE_ROOT, file) };
+};
+
+/** Where install.js puts a target, and where run.js looks for it. */
+export const installDirOf = (target, prefix) => join(prefix, `${target.cli}-${target.version}`);
+
+const expand = (value) => String(value)
+ .replace(/^~(?=$|\/)/, homedir())
+ .replace(/\$\{(\w+)\}/g, (_, name) => process.env[name] ?? '');
+
+/**
+ * The binary a target runs, as `{ path, label }` (label is prefix-relative so a
+ * published page never carries a host's home directory), `null` when the
+ * target names none, or `{ missing }` when it cannot run on this host.
+ */
+export const resolveBinary = (target, { prefix, platform = platformKey() } = {}) => {
+ const binary = target.binary;
+ if (!binary) return null;
+ if (binary.path) return { path: expand(binary.path), label: binary.path };
+ const bin = typeof binary.bin === 'string' ? binary.bin : binary.bin?.[platform];
+ if (!bin) return { missing: `no ${target.cli} binary is pinned for ${platform}` };
+ if (binary.release && !binary.release[platform]) return { missing: `no ${target.cli} release asset is pinned for ${platform}` };
+ if (!prefix) return { missing: 'no install prefix given (--prefix or COMPAT_PREFIX)' };
+ return { path: join(installDirOf(target, expand(prefix)), bin), label: `/${target.cli}-${target.version}/${bin}` };
+};
+
+/**
+ * @returns {Promise<{ adapter } | { skip: string } | { error: string }>}
+ */
+export const loadAdapter = async (target, { binaryPath } = {}) => {
+ let spec;
+ try { spec = parseAdapterSpec(target.adapter); } catch (error) { return { error: error.message }; }
+ if (!existsSync(spec.path)) {
+ const why = target.requires ? ` (requires: ${target.requires})` : '';
+ return { skip: `adapter module ${spec.file} is not present${why}` };
+ }
+ let mod;
+ try { mod = await import(pathToFileURL(spec.path).href); } catch (error) {
+ return { error: `importing ${spec.file} failed: ${error?.message || error}` };
+ }
+ const exported = mod[spec.exportName];
+ if (exported === undefined) return { error: `${spec.file} has no export ${spec.exportName}` };
+ let adapter = exported;
+ if (typeof exported === 'function') {
+ try {
+ adapter = await exported({ ...(target.options || {}), ...(binaryPath ? { binaryPath } : {}) });
+ } catch (error) {
+ return { error: `${spec.file}#${spec.exportName}(options) threw: ${error?.message || error}` };
+ }
+ }
+ if (!adapter || typeof adapter.spawn !== 'function' || typeof adapter.detect !== 'function') {
+ return { error: `${target.adapter} is not an adapter (needs detect() and spawn())` };
+ }
+ return { adapter };
+};
+
+export const targetLabel = (target) => `${target.cli}@${target.version}`;
diff --git a/packages/seats/compat/targets.json b/packages/seats/compat/targets.json
new file mode 100644
index 0000000000..b34dd846e7
--- /dev/null
+++ b/packages/seats/compat/targets.json
@@ -0,0 +1,260 @@
+[
+ {
+ "cli": "stub",
+ "version": "0.0.0",
+ "adapter": "src/adapters/stub.js#default",
+ "enabled": true,
+ "note": "reference no-op adapter: it never calls a provider, so the provider checks fail by design",
+ "timeouts": {
+ "turnMs": 5000,
+ "stallMs": 2000
+ }
+ },
+ {
+ "cli": "fake-cli",
+ "version": "1.0.0",
+ "adapter": "test-support/fake-cli-adapter.js#default",
+ "enabled": true,
+ "note": "a real child process speaking to the fake provider; the runner's own control",
+ "timeouts": {
+ "turnMs": 10000,
+ "stallMs": 2000
+ }
+ },
+ {
+ "cli": "goose",
+ "version": "1.56.0",
+ "adapter": "src/adapters/acp-launch.js#createAcpGooseAdapter",
+ "requires": "acp",
+ "enabled": true,
+ "probeOptions": {
+ "watchProvider": false
+ },
+ "limits": {
+ "retry-status": {
+ "note": "retries a provider 5xx with backoff and sends no session/update while it does; its error text arrives only after it gives up, as an agent_message_chunk followed by end_turn"
+ }
+ },
+ "binary": {
+ "bin": "goose",
+ "release": {
+ "linux-x64": {
+ "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-x86_64-unknown-linux-gnu.tar.gz",
+ "sha256": "225a929b35b5130fb691eafdb50be236f2c0a150019b53973775938f8f19c281"
+ },
+ "linux-arm64": {
+ "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-aarch64-unknown-linux-gnu.tar.gz",
+ "sha256": "26bcb7d3f55aac485a91d81cbdecd8d90e5af1acadeca6e580b4d58b64a888e4"
+ },
+ "darwin-arm64": {
+ "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-aarch64-apple-darwin.tar.gz",
+ "sha256": "52f3f8d62e3d144c5de70c5c471de39d8b92c24d41d5b3436eb8bab11ba4bc24"
+ }
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "opencode",
+ "version": "1.19.4",
+ "adapter": "src/adapters/acp-launch.js#createAcpOpencodeAdapter",
+ "requires": "acp",
+ "enabled": true,
+ "note": "opencode-ai, the 1.x line",
+ "probeOptions": {
+ "watchProvider": false
+ },
+ "limits": {
+ "retry-status": {
+ "note": "retries a provider 5xx with backoff (about 73 s before it gives up) and sends no session/update while it does, then answers session/prompt with -32603 and no HTTP status",
+ "fixedIn": "@opencode/cli ≥2.0.15"
+ }
+ },
+ "binary": {
+ "npm": "opencode-ai@1.19.4",
+ "bin": {
+ "linux-x64": "node_modules/opencode-linux-x64/bin/opencode",
+ "linux-arm64": "node_modules/opencode-linux-arm64/bin/opencode",
+ "darwin-arm64": "node_modules/opencode-darwin-arm64/bin/opencode"
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "opencode-v2",
+ "version": "2.1.4",
+ "adapter": "src/adapters/acp-launch.js#createAcpOpencodeAdapter",
+ "requires": "acp",
+ "enabled": true,
+ "note": "@opencode/cli, the 2.x line: a separate npm package whose launcher needs its postinstall, so the platform binary is run directly",
+ "probeOptions": {
+ "watchProvider": false
+ },
+ "binary": {
+ "npm": "@opencode/cli@2.1.4",
+ "bin": {
+ "linux-x64": "node_modules/@opencode/cli-linux-x64/bin/opencode",
+ "linux-arm64": "node_modules/@opencode/cli-linux-arm64/bin/opencode",
+ "darwin-arm64": "node_modules/@opencode/cli-darwin-arm64/bin/opencode"
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "copilot",
+ "version": "1.0.103",
+ "adapter": "src/adapters/copilot.js#createCopilotAdapter",
+ "enabled": true,
+ "note": "GitHub Copilot CLI, @github/copilot: BYOK through COPILOT_PROVIDER_BASE_URL (type openai) with COPILOT_OFFLINE, so no GitHub sign-in",
+ "binary": {
+ "npm": "@github/copilot@1.0.103",
+ "bin": {
+ "linux-x64": "node_modules/@github/copilot-linux-x64/copilot",
+ "linux-arm64": "node_modules/@github/copilot-linux-arm64/copilot",
+ "darwin-arm64": "node_modules/@github/copilot-darwin-arm64/copilot"
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "gemini",
+ "version": "0.65.0",
+ "adapter": "src/adapters/provider-unsupported.js#createGeminiAdapter",
+ "enabled": true,
+ "note": "Google Gemini CLI, @google/gemini-cli: speaks only the Gemini API, so every turn check reports provider_unsupported",
+ "binary": {
+ "npm": "@google/gemini-cli@0.65.0",
+ "bin": "node_modules/.bin/gemini"
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "qwen-code",
+ "version": "0.26.1",
+ "adapter": "src/adapters/qwen.js#createQwenAdapter",
+ "enabled": true,
+ "note": "Qwen Code, @qwen-code/qwen-code: OpenAI auth through OPENAI_BASE_URL; the production qwen adapter runs the same invocation",
+ "limits": {
+ "provider-5xx": {
+ "note": "retries a provider 503 and has not given up when the 60 s turn deadline ends it (22 requests in 60 s, 0.26.1)"
+ }
+ },
+ "binary": {
+ "npm": "@qwen-code/qwen-code@0.26.1",
+ "bin": "node_modules/.bin/qwen"
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "cline",
+ "version": "3.0.73",
+ "adapter": "src/adapters/cline.js#createClineAdapter",
+ "enabled": true,
+ "note": "Cline CLI, the cline npm package: an openai-compatible provider written to its settings store",
+ "limits": {
+ "provider-5xx": {
+ "note": "retries a provider 503 and has not given up when the 60 s turn deadline ends it (3.0.73)"
+ },
+ "resume-session": {
+ "note": "--id with --json refuses every prompt form (positional, after --, piped stdin) with \"JSON output mode requires a prompt argument or piped stdin\" (3.0.73)"
+ }
+ },
+ "binary": {
+ "npm": "cline@3.0.73",
+ "bin": {
+ "linux-x64": "node_modules/@cline/cli-linux-x64/bin/cline",
+ "linux-arm64": "node_modules/@cline/cli-linux-arm64/bin/cline",
+ "darwin-arm64": "node_modules/@cline/cli-darwin-arm64/bin/cline"
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "aider",
+ "version": "0.86.2",
+ "adapter": "src/adapters/aider.js#createAiderAdapter",
+ "enabled": true,
+ "note": "Aider, aider-chat on PyPI: installed into a venv from wheels only; needs python 3.10 to 3.12 on PATH as python3 (or COMPAT_PYTHON)",
+ "limits": {
+ "refusal": {
+ "note": "litellm reads a 400 content-policy refusal as retryable: aider asks the provider 9 times over about 64 s with backoff, then prints the refusal and stops (0.86.2)"
+ }
+ },
+ "binary": {
+ "pip": "aider-chat==0.86.2",
+ "bin": "venv/bin/aider"
+ },
+ "timeouts": {
+ "turnMs": 120000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "amp",
+ "version": "0.0.1792958443-g695471",
+ "adapter": "src/adapters/provider-unsupported.js#createAmpAdapter",
+ "enabled": true,
+ "note": "Amp, @ampcode/cli: model calls go through the Amp service, so every turn check reports provider_unsupported",
+ "binary": {
+ "npm": "@ampcode/cli@0.0.1792958443-g695471",
+ "bin": {
+ "linux-x64": "node_modules/@ampcode/cli-linux-x64/amp",
+ "linux-arm64": "node_modules/@ampcode/cli-linux-arm64/amp",
+ "darwin-arm64": "node_modules/@ampcode/cli-darwin-arm64/amp"
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ },
+ {
+ "cli": "cursor",
+ "version": "2026.10.22-27dcf5f",
+ "adapter": "src/adapters/provider-unsupported.js#createCursorAdapter",
+ "enabled": true,
+ "note": "Cursor CLI, cursor-agent: model calls go through Cursor's backend, so every turn check reports provider_unsupported",
+ "binary": {
+ "bin": "dist-package/cursor-agent",
+ "release": {
+ "linux-x64": {
+ "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/linux/x64/agent-cli-package.tar.gz",
+ "sha256": "85296944e4600a83c4b764f578d1502bf9c416db200d33a593e8f3ecabfdc294"
+ },
+ "linux-arm64": {
+ "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/linux/arm64/agent-cli-package.tar.gz",
+ "sha256": "22a36df9720830668ccd4d707e5190204ab9c8a0efb2f068cc914895f804d3ec"
+ },
+ "darwin-arm64": {
+ "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/darwin/arm64/agent-cli-package.tar.gz",
+ "sha256": "2f5b0a3dd5fb951cecffac813867d8a08d810e59837d59fd5f12d69b922398d7"
+ }
+ }
+ },
+ "timeouts": {
+ "turnMs": 60000,
+ "stallMs": 15000
+ }
+ }
+]
diff --git a/packages/seats/compat/upstream.js b/packages/seats/compat/upstream.js
new file mode 100644
index 0000000000..60fb219162
--- /dev/null
+++ b/packages/seats/compat/upstream.js
@@ -0,0 +1,102 @@
+/**
+ * Known upstream bugs and what we do about each, as data
+ * (compat/upstream.json): `{ id, cli, versions, bug, status, workaround:
+ * { name, ref, where, tests[] }, fixed?: { version, verified },
+ * matrix?: { target, check }, references[] }`.
+ *
+ * We do not wait on upstream. Every row is "worked around", "fixed upstream"
+ * or "open", and the rule is enforced here, not trusted from the file: a row
+ * reads as worked around only when its workaround names the pushed commit it
+ * lives on (`ref`, a full sha) and at least one test there that proves it. A
+ * row reads as fixed upstream only when it names the release that fixed it
+ * (`fixed.version`), says how we verified that release ourselves
+ * (`fixed.verified`), and links the upstream issue. Anything less is open,
+ * with the planned workaround named. An upstream issue is a reference, never
+ * the proof, so a link alone changes nothing.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { isLimitRow } from './render.js';
+
+export const UPSTREAM_STATUSES = Object.freeze(['worked around', 'fixed upstream', 'open']);
+
+const text = (value) => (typeof value === 'string' ? value.trim() : '');
+const SHA = /^[0-9a-f]{40}$/;
+
+const HTTPS = /^https:\/\/\S+$/;
+
+/** "worked around" only with a commit and a test; otherwise "open". */
+const workedAround = (row) => {
+ const workaround = row?.workaround || {};
+ return SHA.test(text(workaround.ref)) && Array.isArray(workaround.tests) && workaround.tests.some((t) => text(t));
+};
+
+/** "fixed upstream" only with the fixing release, our own check of it, and the issue link. */
+const fixedUpstream = (row) => Boolean(text(row?.fixed?.version) && text(row?.fixed?.verified)
+ && Array.isArray(row?.references) && row.references.some((r) => HTTPS.test(text(r))));
+
+export const upstreamStatusOf = (row) => {
+ if (row?.status === 'worked around' && workedAround(row)) return 'worked around';
+ if (row?.status === 'fixed upstream' && fixedUpstream(row)) return 'fixed upstream';
+ return 'open';
+};
+
+const DOWNGRADED = {
+ 'worked around': 'listed as worked around, but names no commit or no test that proves it',
+ 'fixed upstream': 'listed as fixed upstream, but names no fixing release, no check of it, or no upstream link',
+};
+
+/**
+ * Normalise rows for the results document. A row that claims "worked around"
+ * without the proof is downgraded and says why, so the page never repeats an
+ * unproven claim.
+ */
+export const normaliseUpstream = (rows) => (Array.isArray(rows) ? rows : []).map((row) => {
+ const status = upstreamStatusOf(row);
+ const workaround = row?.workaround || {};
+ return {
+ id: text(row?.id),
+ cli: text(row?.cli),
+ versions: text(row?.versions),
+ bug: text(row?.bug),
+ status,
+ ...(DOWNGRADED[row?.status] && status === 'open' ? { downgraded: DOWNGRADED[row.status] } : {}),
+ workaround: {
+ name: text(workaround.name),
+ ref: text(workaround.ref),
+ where: text(workaround.where),
+ tests: (Array.isArray(workaround.tests) ? workaround.tests : []).map(text).filter(Boolean),
+ },
+ ...(status === 'fixed upstream' ? { fixed: { version: text(row.fixed.version), verified: text(row.fixed.verified) } } : {}),
+ ...(row?.matrix?.target && row?.matrix?.check ? { matrix: { target: text(row.matrix.target), check: text(row.matrix.check) } } : {}),
+ references: (Array.isArray(row?.references) ? row.references : []).map(text).filter(Boolean),
+ };
+});
+
+/**
+ * What the matrix saw for the check a row points at, so a "worked around"
+ * claim sits next to this run's evidence: pass, fail, limit, skip, or
+ * "not run" when no row matches.
+ */
+export const observedOf = (pointer, rows) => {
+ if (!pointer) return null;
+ const row = (Array.isArray(rows) ? rows : []).find((r) => r.target === pointer.target && r.check === pointer.check);
+ if (!row) return 'not run';
+ return isLimitRow(row) ? 'limit' : row.status;
+};
+
+export const upstreamForResults = (rows, matrixRows) => normaliseUpstream(rows)
+ .map((row) => (row.matrix ? { ...row, matrix: { ...row.matrix, observed: observedOf(row.matrix, matrixRows) } } : row));
+
+/** Validate the file; the rows are normalised when they join a results document. */
+export const loadUpstream = (path) => {
+ if (!path || !existsSync(path)) return [];
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
+ if (!Array.isArray(parsed)) throw new Error(`${path} must hold an array of upstream rows`);
+ for (const row of parsed) {
+ if (!row?.id || !row?.cli || !row?.bug || !row?.workaround?.name) {
+ throw new Error(`every upstream row needs id, cli, bug and workaround.name: ${JSON.stringify(row)}`);
+ }
+ if (!UPSTREAM_STATUSES.includes(row.status)) throw new Error(`upstream row ${row.id} has status ${JSON.stringify(row.status)}; use "worked around", "fixed upstream" or "open"`);
+ }
+ return parsed;
+};
diff --git a/packages/seats/compat/upstream.json b/packages/seats/compat/upstream.json
new file mode 100644
index 0000000000..e3d6ca1292
--- /dev/null
+++ b/packages/seats/compat/upstream.json
@@ -0,0 +1,126 @@
+[
+ {
+ "id": "goose-refusal-retried",
+ "cli": "goose",
+ "versions": "1.54.0 to 1.56.0",
+ "bug": "retries a 400 content-policy refusal 3 times with a byte-identical body, for both its title call and its reply call",
+ "status": "worked around",
+ "workaround": {
+ "name": "fail fast on the first refusal on the wire, as RunFailure refusal_retry_loop (provider watch in the ACP adapter)",
+ "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf",
+ "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js",
+ "tests": [
+ "packages/seats/test/provider-watch.test.js: the first refusal trips once, its body reaches the CLI, and nothing more is forwarded",
+ "packages/seats/test/acp.test.js: the first provider refusal on the wire ends the run as refusal_retry_loop, fast",
+ "packages/seats/test/acp-real.test.js: goose acp: the first provider refusal ends the run as refusal_retry_loop"
+ ]
+ },
+ "matrix": {
+ "target": "goose@1.56.0",
+ "check": "refusal"
+ },
+ "references": []
+ },
+ {
+ "id": "goose-5xx-silent",
+ "cli": "goose",
+ "versions": "1.56.0",
+ "bug": "over ACP, retries a provider 5xx with backoff and sends no session/update while it does; the error reaches the client only after it gives up, as message text ending in end_turn",
+ "status": "worked around",
+ "workaround": {
+ "name": "our own 5xx detection: the first 5xx on the wire ends the run as RunFailure provider_unavailable (provider watch in the ACP adapter)",
+ "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf",
+ "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js",
+ "tests": [
+ "packages/seats/test/provider-watch.test.js: the first 503 trips as provider_unavailable",
+ "packages/seats/test/acp-real.test.js: goose acp: the first provider 503 ends the run as provider_unavailable"
+ ]
+ },
+ "matrix": {
+ "target": "goose@1.56.0",
+ "check": "provider-5xx"
+ },
+ "references": []
+ },
+ {
+ "id": "codex-refusal-exit-0",
+ "cli": "codex",
+ "versions": "0.162.1 measured; 0.164.0 reported",
+ "bug": "an in-band provider refusal ends `codex exec --json` with exit 0 and no output, so the run looks like an empty success",
+ "status": "open",
+ "workaround": {
+ "name": "refusal_dropped detector over the provider response bodies, fed by a per-run loopback provider relay",
+ "ref": "077a9a12c04429a1ffb6677cb17821368a0c8cef",
+ "where": "packages/seats/src/wire/refusal.js (the detector, with its unit tests); the per-run relay that feeds it is not built, so it cannot fire on a live run yet",
+ "tests": []
+ },
+ "references": [
+ "https://github.com/openai/codex/issues/56875"
+ ]
+ },
+ {
+ "id": "opencode-5xx-silent",
+ "cli": "opencode",
+ "versions": "opencode-ai 1.19.0 to 1.19.4",
+ "bug": "over ACP, retries a provider 5xx for about 73 s with no session/update, then answers -32603 with no HTTP status",
+ "status": "worked around",
+ "workaround": {
+ "name": "our own 5xx detection: the first 5xx on the wire ends the run as RunFailure provider_unavailable (provider watch in the ACP adapter); pinning @opencode/cli 2.x is the second route, see its row",
+ "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf",
+ "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js",
+ "tests": [
+ "packages/seats/test/provider-watch.test.js: the first 503 trips as provider_unavailable",
+ "packages/seats/test/acp.test.js: the first provider 5xx on the wire ends the run as provider_unavailable, fast",
+ "packages/seats/test/acp-real.test.js: opencode acp: the first provider 503 ends the run as provider_unavailable, not at a timeout"
+ ]
+ },
+ "matrix": {
+ "target": "opencode@1.19.4",
+ "check": "provider-5xx"
+ },
+ "references": [
+ "https://github.com/anomalyco/opencode/issues/50743",
+ "https://github.com/anomalyco/opencode/pull/50752"
+ ]
+ },
+ {
+ "id": "opencode-v2-retry-status-unread",
+ "cli": "opencode-v2",
+ "versions": "@opencode/cli 2.1.4",
+ "bug": "not an upstream bug: 2.x forwards each retry as a session_info_update whose only content is _meta[\"opencode/retry\"], and our ACP adapter drops that update kind, so the trace shows none of it",
+ "status": "open",
+ "workaround": {
+ "name": "read session_info_update._meta[\"opencode/retry\"] in the ACP adapter's update mapping",
+ "ref": "",
+ "where": "packages/seats/src/adapters/acp.js (traceUpdate), not started",
+ "tests": []
+ },
+ "matrix": {
+ "target": "opencode-v2@2.1.4",
+ "check": "retry-status-surfaced"
+ },
+ "references": []
+ },
+ {
+ "id": "pi-cowboy-explore-hang",
+ "cli": "pi-cowboy",
+ "versions": "0.22.1 and earlier",
+ "bug": "an explore agent may not write its result file, because its prompt forbids creating files; a foreground call then never settles",
+ "status": "fixed upstream",
+ "fixed": {
+ "version": "0.22.2",
+ "verified": "6 of 6 fresh-session runs on 0.22.2 settled (subagent done in 12 to 17 s, parent idle in 16 to 21 s), in a clean node:22-bookworm container with pi 1.4.0, herdr 0.9.5 and pi-cowboy as the only plugin; the same repro on 0.22.1 hung about one run in three"
+ },
+ "workaround": {
+ "name": "settle a run that ended without a report, in our pi-cowboy plugin; kept for 0.22.1 and earlier, since 0.22.2 still settles only on a report",
+ "ref": "65ccaf5444f9a34d9d5ebfe28217c863305b8ca8",
+ "where": "pi-cowboy-commonly 0.1.2 on npm (Team-Commonly/pi-cowboy-commonly, src/report-guard.js). Real-model check on pi-cowboy 0.22.1: 6 of 6 runs settled in 11 to 15 s on deepseek-v4-flash (3 wrote a report, 3 got the missing-report notice), none hung",
+ "tests": [
+ "pi-cowboy-commonly test/report-guard.test.js: KEY: an explore subagent that settles without a report gets a missing-report notice"
+ ]
+ },
+ "references": [
+ "https://github.com/electric-dragon-80000v/pi-cowboy/issues/33"
+ ]
+ }
+]
diff --git a/packages/seats/src/adapters/aider.js b/packages/seats/src/adapters/aider.js
new file mode 100644
index 0000000000..2bc412a8a6
--- /dev/null
+++ b/packages/seats/src/adapters/aider.js
@@ -0,0 +1,85 @@
+/**
+ * Aider, one turn per process:
+ * aider --model openai/ --openai-api-base --message --no-stream …
+ *
+ * Aider reaches the provider through litellm's OpenAI route; the key goes in
+ * OPENAI_API_KEY, never argv. It has no machine-readable output: stdout is a
+ * banner, the reply, then a `Tokens: … sent, … received.` line. A provider
+ * error prints in place of the reply, with no Tokens line, and aider still
+ * exits 0, so the Tokens line is what says the turn answered.
+ *
+ * Aider has no session ids. A session here is a chat-history file under the
+ * run's HOME, named by an id the adapter makes; a resume restores it with
+ * --restore-chat-history.
+ */
+import { randomUUID } from 'node:crypto';
+import { mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { runFailure } from '../failures.js';
+import { createHeadlessAdapter, failureFromText } from './headless.js';
+
+const SESSION_ID = /^[A-Za-z0-9-]{8,64}$/;
+const TOKENS_LINE = /^Tokens: [\d.,]+k? sent/;
+// What aider prints before the reply; none of it is the agent's answer.
+const BANNER_LINE = /^(?:Aider v\d|Model: |Weak model: |Editor model: |Git repo: |Repo-map: |Restored previous conversation history|Added .* to the chat|https:\/\/aider\.chat\/|Warning: |Note: |Cur working dir: |HTTPSConnectionPool\(|Can't initialize prompt toolkit)/;
+
+export const readAiderLine = (line, state) => {
+ (state.lines ||= []).push(line);
+};
+
+/** The reply is what sits between the banner and the last Tokens line. */
+export const settleAider = (state, { code, stderr, credentials }) => {
+ const lines = state.lines || [];
+ let end = -1;
+ for (let i = lines.length - 1; i >= 0; i -= 1) if (TOKENS_LINE.test(lines[i])) { end = i; break; }
+ let start = 0;
+ for (let i = 0; i < (end === -1 ? lines.length : end); i += 1) if (BANNER_LINE.test(lines[i])) start = i + 1;
+ if (end === -1) {
+ const said = lines.slice(start).filter((l) => l.trim() && !/^Retrying in /.test(l)).join(' ');
+ throw failureFromText('aider', said || stderr || `exit ${code} with no reply`, { credentials, exitCode: code });
+ }
+ if (code !== 0) throw failureFromText('aider', stderr || `exit ${code}`, { credentials, exitCode: code });
+ const text = lines.slice(start, end).join('\n').trim();
+ if (!text) throw runFailure('protocol', 'aider printed a Tokens line and no reply');
+ return { text, newSessionId: state.sessionId };
+};
+
+export const createAiderAdapter = createHeadlessAdapter({
+ name: 'aider',
+ cli: 'aider',
+ command: 'aider',
+ versionPattern: /aider\s+v?(\d+\.\d+\.\d+[^\s]*)/,
+ async launch(prompt, ctx, { provider, env, home }) {
+ if (ctx.sessionId && !SESSION_ID.test(ctx.sessionId)) throw runFailure('unsupported', 'aider: session id is not one this adapter made');
+ const sessionId = ctx.sessionId || randomUUID();
+ const dir = join(home, '.seats-aider');
+ await mkdir(dir, { recursive: true, mode: 0o700 });
+ return {
+ args: [
+ '--model', `openai/${provider.model}`, '--openai-api-base', provider.baseURL,
+ '--message', prompt,
+ '--chat-history-file', join(dir, `${sessionId}.chat.md`),
+ '--input-history-file', join(dir, `${sessionId}.input`),
+ ...(ctx.sessionId ? ['--restore-chat-history'] : ['--no-restore-chat-history']),
+ '--llm-history-file', '/dev/null',
+ '--yes-always', '--no-git', '--no-gitignore', '--no-auto-commits', '--no-auto-lint',
+ '--no-pretty', '--no-stream', '--no-fancy-input', '--no-detect-urls', '--no-suggest-shell-commands',
+ '--no-check-update', '--no-analytics', '--no-show-model-warnings', '--no-show-release-notes',
+ '--map-tokens', '0',
+ ],
+ env: {
+ ...env,
+ OPENAI_API_KEY: provider.apiKey,
+ // litellm would otherwise fetch its model price list from GitHub on start.
+ LITELLM_LOCAL_MODEL_COST_MAP: 'True',
+ AIDER_ANALYTICS: 'false',
+ AIDER_CHECK_UPDATE: 'false',
+ },
+ state: { sessionId },
+ };
+ },
+ read: readAiderLine,
+ settle: settleAider,
+});
+
+export default createAiderAdapter;
diff --git a/packages/seats/src/adapters/cline.js b/packages/seats/src/adapters/cline.js
new file mode 100644
index 0000000000..7e3cb44384
--- /dev/null
+++ b/packages/seats/src/adapters/cline.js
@@ -0,0 +1,106 @@
+/**
+ * Cline CLI, one headless turn per process:
+ * cline --json --config -P openai-compatible -m [--id ]
+ *
+ * Cline takes a provider only from its settings store (`cline auth` wants the
+ * key in argv), so the adapter writes /data/settings/providers.json
+ * itself, mode 0600, with the run's base URL and key. The config directory
+ * lives under the run's HOME, so a resume finds the task the first turn made.
+ * The prompt is the one positional argument; the key is never in argv.
+ *
+ * stdout is NDJSON: agent_event content_start carries text, and run_result
+ * closes the turn with finishReason and text. A failure also arrives on
+ * stderr as {"type":"error","message":...}.
+ *
+ * The id `--id` resumes is the session id `cline history` lists, which
+ * stdout never prints (its taskId is a different id). Each turn records one
+ * directory under /data/sessions/ named by it, so the session id is
+ * the one directory the turn added. Measured on 3.0.73: `--id` with `--json`
+ * then refuses every prompt form (positional, after `--`, piped stdin) with
+ * "JSON output mode requires a prompt argument or piped stdin", so a resume
+ * fails by name; targets.json carries that as a limit.
+ */
+import { readdirSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHeadlessAdapter, failureFromText } from './headless.js';
+import { runFailure } from '../failures.js';
+
+const PROVIDER = 'openai-compatible';
+
+const parse = (line) => {
+ try { return JSON.parse(line); } catch { return null; }
+};
+
+export const readClineEvent = (line, state) => {
+ const event = parse(line);
+ if (!event || typeof event !== 'object') return;
+ if (event.type === 'agent_event') {
+ const inner = event.event || {};
+ if (inner.type === 'content_start' && inner.contentType === 'text' && typeof inner.text === 'string' && inner.text) state.chunks.push(inner.text);
+ if (inner.type === 'error') state.lastError = String(inner.error?.message || 'agent error');
+ } else if (event.type === 'run_result') {
+ if (event.finishReason === 'completed') state.text = String(event.text ?? '');
+ else state.error = String(event.text || state.lastError || `cline finished with ${event.finishReason}`);
+ } else if (event.type === 'error' && !state.error) {
+ state.error = String(event.message || 'cline error');
+ }
+};
+
+// Cline ignores a provider entry with no updatedAt and falls back to its own
+// default route (api.openai.com, or api.cline.bot with no -P), measured on
+// 3.0.73: the file `cline auth` writes differs from this one only by it.
+export const providersFile = ({ baseURL, apiKey, model }, now = new Date()) => ({
+ version: 1,
+ lastUsedProvider: PROVIDER,
+ modes: {},
+ providers: {
+ [PROVIDER]: {
+ settings: { provider: PROVIDER, apiKey, model, baseUrl: baseURL },
+ updatedAt: now.toISOString(),
+ tokenSource: 'manual',
+ },
+ },
+});
+
+const sessionsOf = (dir) => {
+ try { return readdirSync(dir); } catch { return []; }
+};
+
+export const settleCline = (state, { code, stderr, credentials }) => {
+ if (state.error || code !== 0) {
+ throw failureFromText('cline', state.error || stderr || `exit ${code}`, { credentials, exitCode: code });
+ }
+ if (typeof state.text !== 'string') throw runFailure('protocol', 'cline exited 0 without a run_result');
+ const before = new Set(state.sessionsBefore || []);
+ const added = state.sessionsDir ? sessionsOf(state.sessionsDir).filter((name) => !before.has(name)) : [];
+ const sessionId = state.resumed || (added.length === 1 ? added[0] : null);
+ return { text: state.text, ...(sessionId ? { newSessionId: sessionId } : {}) };
+};
+
+export const createClineAdapter = createHeadlessAdapter({
+ name: 'cline',
+ cli: 'cline',
+ command: 'cline',
+ versionPattern: /(\d+\.\d+\.\d+[^\s]*)/,
+ async launch(prompt, ctx, { provider, env, home }) {
+ const config = join(home, '.cline');
+ const settings = join(config, 'data', 'settings');
+ await mkdir(settings, { recursive: true, mode: 0o700 });
+ await writeFile(join(settings, 'providers.json'), JSON.stringify(providersFile(provider), null, 2), { mode: 0o600 });
+ const sessionsDir = join(config, 'data', 'sessions');
+ return {
+ state: { sessionsDir, sessionsBefore: sessionsOf(sessionsDir), resumed: ctx.sessionId || null },
+ args: [
+ '--json', '--config', config, '-P', PROVIDER, '-m', provider.model,
+ ...(ctx.sessionId ? ['--id', ctx.sessionId] : []),
+ '--', prompt,
+ ],
+ env,
+ };
+ },
+ read: readClineEvent,
+ settle: settleCline,
+});
+
+export default createClineAdapter;
diff --git a/packages/seats/src/adapters/copilot.js b/packages/seats/src/adapters/copilot.js
new file mode 100644
index 0000000000..f38a56729e
--- /dev/null
+++ b/packages/seats/src/adapters/copilot.js
@@ -0,0 +1,71 @@
+/**
+ * GitHub Copilot CLI, one headless turn per process:
+ * copilot --output-format json --allow-all-tools [--resume ] < prompt
+ *
+ * Copilot reaches the run's provider through its BYOK variables
+ * (COPILOT_PROVIDER_BASE_URL with type openai). With COPILOT_OFFLINE set it
+ * makes no GitHub call at all, so no GitHub sign-in is needed. The key goes
+ * in COPILOT_PROVIDER_API_KEY, never argv; the prompt goes on stdin.
+ *
+ * stdout is JSONL: assistant.message_delta / assistant.message carry the
+ * reply, session.error carries a provider failure with its HTTP status, and
+ * the closing `result` event carries the session id a resume names.
+ */
+import { join } from 'node:path';
+import { createHeadlessAdapter } from './headless.js';
+
+const parse = (line) => {
+ try { return JSON.parse(line); } catch { return null; }
+};
+
+export const readCopilotEvent = (line, state) => {
+ const event = parse(line);
+ if (!event || typeof event !== 'object') return;
+ const data = event.data || {};
+ switch (event.type) {
+ case 'assistant.message_delta':
+ if (typeof data.deltaContent === 'string' && data.deltaContent) state.chunks.push(data.deltaContent);
+ break;
+ case 'assistant.message':
+ // A turn with tool calls has several assistant messages; the reply is
+ // the last one that says something.
+ if (typeof data.content === 'string' && data.content) state.text = data.content;
+ break;
+ case 'session.error':
+ state.error = String(data.message || data.errorType || 'session.error');
+ break;
+ case 'result':
+ if (typeof event.sessionId === 'string' && event.sessionId) state.sessionId = event.sessionId;
+ break;
+ default:
+ }
+};
+
+export const createCopilotAdapter = createHeadlessAdapter({
+ name: 'copilot',
+ cli: 'copilot',
+ command: 'copilot',
+ versionPattern: /Copilot CLI\s+v?(\d+\.\d+\.\d+[^\s.]*)/,
+ async launch(prompt, ctx, { provider, env, home }) {
+ return {
+ args: [
+ '--output-format', 'json', '--allow-all-tools',
+ ...(ctx.sessionId ? [`--resume=${ctx.sessionId}`] : []),
+ ],
+ stdin: prompt,
+ env: {
+ ...env,
+ COPILOT_HOME: join(home, '.copilot'),
+ COPILOT_OFFLINE: 'true',
+ COPILOT_AUTO_UPDATE: 'false',
+ COPILOT_PROVIDER_TYPE: 'openai',
+ COPILOT_PROVIDER_BASE_URL: provider.baseURL,
+ COPILOT_PROVIDER_API_KEY: provider.apiKey,
+ COPILOT_MODEL: provider.model,
+ },
+ };
+ },
+ read: readCopilotEvent,
+});
+
+export default createCopilotAdapter;
diff --git a/packages/seats/src/adapters/headless.js b/packages/seats/src/adapters/headless.js
new file mode 100644
index 0000000000..25e960a6a6
--- /dev/null
+++ b/packages/seats/src/adapters/headless.js
@@ -0,0 +1,269 @@
+/**
+ * One adapter shape for agent CLIs that run one turn per process in their own
+ * headless mode (`copilot -p`, `qwen -o stream-json`, `aider --message`, …).
+ * The CLI-specific part is a `launch(prompt, ctx)` that returns argv and env,
+ * a `read(event, state)` for each stdout line, and a `settle(state)` that
+ * turns what was read into a reply. Everything else is here, once: the
+ * provider route check, an isolated HOME, the process group, the turn
+ * deadline, cancellation, the trace and the scrubbed failure text.
+ *
+ * The provider key travels in the child env or a 0600 file, never in argv,
+ * which any same-user process can read.
+ */
+import { spawn as childSpawn, spawnSync } from 'node:child_process';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
+import { RunFailure, runFailure } from '../failures.js';
+import { observeSpawn } from '../runtime-observation.js';
+import { traceOf } from '../trace/events.js';
+import { scrubAdapterFailure } from '../upstream-refusal.js';
+
+const SAFE_ENV = /^(?:PATH|LANG|LC_[A-Z_]+|TERM|USER|LOGNAME|NO_COLOR|CI|TMPDIR|SSL_CERT_FILE|SSL_CERT_DIR)$/;
+const VERSION_TIMEOUT_MS = 15000;
+const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000;
+const KILL_GRACE_MS = 2000;
+const MAX_STDERR_CHARS = 8192;
+const MAX_LINE_CHARS = 1024 * 1024;
+const FAILURE_LIMIT = 500;
+const REFUSAL_WORDING = /content[ _-]?policy|refus|rejected by/i;
+const OWN_GROUP = process.platform !== 'win32';
+
+/** The provider route, checked: a plain HTTP(S) base URL, a key and a model. */
+export const providerOf = (ctx, label) => {
+ const { baseURL, apiKey, model } = ctx?.provider || {};
+ let url;
+ try { url = new URL(String(baseURL)); } catch {
+ throw runFailure('unsupported', `${label}: ctx.provider.baseURL must be an HTTP(S) URL`);
+ }
+ if (!/^https?:$/.test(url.protocol) || url.username || url.password || url.search || url.hash) {
+ throw runFailure('unsupported', `${label}: ctx.provider.baseURL must be a plain HTTP(S) URL with no credentials, query or fragment`);
+ }
+ if (typeof apiKey !== 'string' || !apiKey) throw runFailure('unsupported', `${label}: ctx.provider.apiKey is missing`);
+ if (typeof model !== 'string' || !model) throw runFailure('unsupported', `${label}: ctx.provider.model is missing`);
+ return { url, baseURL: url.href.replace(/\/$/, ''), apiKey, model };
+};
+
+/**
+ * Safe launch variables plus HOME and XDG roots. HOME comes from ctx.env when
+ * the caller gives one, because a resume reads the sessions kept there;
+ * without one the spawn gets a throwaway HOME that is removed afterwards.
+ */
+export const isolatedEnv = async (ctx, label) => {
+ const source = ctx.env || process.env;
+ const env = Object.fromEntries(Object.entries(source).filter(([name]) => SAFE_ENV.test(name)));
+ const callerHome = ctx.env?.HOME;
+ const home = callerHome || await mkdtemp(join(tmpdir(), `seats-${label}-home-`));
+ env.HOME = home;
+ env.XDG_CONFIG_HOME = ctx.env?.XDG_CONFIG_HOME || join(home, '.config');
+ env.XDG_DATA_HOME = ctx.env?.XDG_DATA_HOME || join(home, '.local', 'share');
+ env.XDG_STATE_HOME = ctx.env?.XDG_STATE_HOME || join(home, '.local', 'state');
+ env.XDG_CACHE_HOME = ctx.env?.XDG_CACHE_HOME || join(home, '.cache');
+ env.NO_COLOR = '1';
+ const removeHome = callerHome ? async () => {} : () => rm(home, { recursive: true, force: true });
+ return { env, home, removeHome };
+};
+
+/** ` --version`, read with `pattern`; null when the binary is absent or fails. */
+export const readVersion = (binary, pattern, { args = ['--version'], env = process.env, spawnSyncImpl = spawnSync } = {}) => {
+ const probe = spawnSyncImpl(binary, args, { encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS });
+ if (probe.error || probe.status !== 0) return null;
+ const version = pattern.exec(`${probe.stdout || ''}\n${probe.stderr || ''}`)?.[1];
+ return version ? { path: binary, version } : null;
+};
+
+/**
+ * A CLI's own failure text, as a named failure: a 4xx or refusal wording is a
+ * refusal (never retried), anything else is the process ending unsettled.
+ */
+export const failureFromText = (label, text, { credentials = [], exitCode = null } = {}) => {
+ const raw = String(text ?? '').trim();
+ const { status, detail } = scrubAdapterFailure(raw, { credentials, limit: FAILURE_LIMIT });
+ const refusal = (status !== null && status >= 400 && status <= 499) || REFUSAL_WORDING.test(raw);
+ const code = exitCode === null ? '' : ` (exit ${exitCode})`;
+ const upstream = status === null ? '' : ` upstream ${status}`;
+ return runFailure(refusal ? 'refusal' : 'exited', `${label}${code}${upstream}: ${detail || 'no error text'}`, {
+ credentials, limit: FAILURE_LIMIT,
+ });
+};
+
+const signal = (child, sig) => {
+ if (OWN_GROUP && child.pid !== undefined) {
+ try { process.kill(-child.pid, sig); return; } catch { /* group gone; fall through */ }
+ }
+ try { child.kill(sig); } catch { /* already gone */ }
+};
+
+/**
+ * @param {object} spec
+ * @param {string} spec.name adapter name
+ * @param {string} spec.cli CLI name in traces and failures
+ * @param {string} spec.command the command when no binaryPath is given
+ * @param {RegExp} spec.versionPattern reads the version out of `--version`
+ * @param {string[]} [spec.versionArgs]
+ * @param {(prompt: string, ctx: object, run: { command: string, provider: object, env: object, home: string }) =>
+ * Promise<{ args: string[], env?: object, stdin?: string, state?: object, cleanup?: () => Promise }>} spec.launch
+ * (`state` seeds what read() and settle() see, e.g. a session id the adapter made)
+ * @param {(line: string, state: object) => void} spec.read one stdout line; may set state.text,
+ * state.sessionId, state.error, state.chunks (pushed text, traced as agent.message_chunk)
+ * @param {(state: object, end: { code: number|null, stderr: string, credentials: string[] }) =>
+ * { text: string, newSessionId?: string }} [spec.settle]
+ */
+export const createHeadlessAdapter = (spec) => ({ binaryPath } = {}) => {
+ const command = binaryPath || spec.command;
+ const versionArgs = spec.versionArgs || ['--version'];
+ const settle = spec.settle || ((state, { code, stderr, credentials }) => {
+ if (state.error || code !== 0) {
+ throw failureFromText(spec.cli, state.error || stderr || `exit ${code}`, { credentials, exitCode: code });
+ }
+ if (typeof state.text !== 'string') throw runFailure('protocol', `${spec.cli} exited 0 without a reply`);
+ return { text: state.text, ...(state.sessionId ? { newSessionId: state.sessionId } : {}) };
+ });
+
+ return {
+ name: spec.name,
+ runtimeType: spec.name,
+
+ async detect() {
+ return readVersion(command, spec.versionPattern, { args: versionArgs });
+ },
+
+ async spawn(prompt, ctx = {}) {
+ const trace = traceOf(ctx);
+ const started = Date.now();
+ const settled = (outcome, failure) => trace.emit('run.settled', {
+ outcome, durationMs: Date.now() - started, attempts: 1,
+ ...(failure ? { failure: { kind: failure.kind, detail: failure.detail } } : {}),
+ });
+ trace.emit('run.started', { adapter: spec.name, cli: { name: spec.cli }, sessionId: ctx.sessionId, attempt: 1 });
+ let launched;
+ let removeHome = async () => {};
+ try {
+ const provider = providerOf(ctx, spec.cli);
+ const isolated = await isolatedEnv(ctx, spec.cli);
+ removeHome = isolated.removeHome;
+ launched = await spec.launch(String(prompt ?? ''), ctx, { command, provider, env: isolated.env, home: isolated.home });
+ } catch (error) {
+ await removeHome();
+ const failure = error instanceof RunFailure ? error : runFailure('unsupported', `${spec.cli}: ${error?.message || error}`);
+ settled('failed', failure);
+ throw failure;
+ }
+ const credentials = [ctx.provider?.apiKey, ctx.runtimeToken].filter((v) => typeof v === 'string' && v.length >= 8);
+ const requested = Number(ctx.timeoutMs);
+ const timeoutMs = Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_TIMEOUT_MS;
+ trace.emit('turn.prompt_sent', { chars: String(prompt ?? '').length });
+ try {
+ const end = await run({
+ command, launched, ctx, timeoutMs, read: spec.read, trace,
+ });
+ if (end.ended === 'aborted') {
+ throw ctx.signal?.reason instanceof RunFailure ? ctx.signal.reason : runFailure('cancelled', 'the caller cancelled the turn');
+ }
+ if (end.ended === 'timeout') throw runFailure('timeout', `${spec.cli} made no answer within ${timeoutMs}ms`);
+ if (end.ended === 'spawn-error') throw runFailure('exited', `${spec.cli} could not be started: ${end.spawnError}`, { credentials });
+ if (end.ended === 'protocol') throw runFailure('protocol', `${spec.cli}: ${end.protocolError}`, { credentials });
+ const result = settle(end.state, { code: end.code, stderr: end.stderr, credentials });
+ settled('ok');
+ return result;
+ } catch (error) {
+ const failure = error instanceof RunFailure ? error : runFailure('protocol', `${spec.cli}: ${error?.message || error}`, { credentials });
+ settled(failure.kind === 'cancelled' ? 'cancelled' : 'failed', failure);
+ throw failure;
+ } finally {
+ await launched.cleanup?.().catch(() => {});
+ await removeHome().catch(() => {});
+ }
+ },
+ };
+};
+
+const run = ({ command, launched, ctx, timeoutMs, read, trace }) => new Promise((resolve) => {
+ const spawnImpl = ctx._spawnImpl || childSpawn;
+ const child = spawnImpl(command, launched.args, {
+ cwd: ctx.cwd, env: launched.env, stdio: ['pipe', 'pipe', 'pipe'], detached: OWN_GROUP,
+ });
+ observeSpawn(child, ctx.onSpawn, launched.controls || {});
+ const state = { ...(launched.state || {}), chunks: [] };
+ const decoder = new StringDecoder('utf8');
+ let buffer = '';
+ let stderr = '';
+ let ended = null;
+ let protocolError = null;
+ let spawnError = null;
+ let killTimer;
+ let closed = false;
+ const stop = (why) => {
+ if (ended || closed) return;
+ ended = why;
+ signal(child, 'SIGTERM');
+ killTimer = setTimeout(() => { if (!closed) signal(child, 'SIGKILL'); }, KILL_GRACE_MS);
+ killTimer.unref?.();
+ };
+ const timer = setTimeout(() => stop('timeout'), timeoutMs);
+ const onAbort = () => stop('aborted');
+ ctx.signal?.addEventListener('abort', onAbort, { once: true });
+ if (ctx.signal?.aborted) onAbort();
+ const take = (line) => {
+ if (!line || ended) return;
+ const before = state.chunks.length;
+ try { read(line, state); } catch (error) {
+ protocolError = error?.message || String(error);
+ stop('protocol');
+ return;
+ }
+ for (const text of state.chunks.slice(before)) trace.emit('agent.message_chunk', { text });
+ };
+ child.stdout.on('data', (chunk) => {
+ buffer += decoder.write(chunk);
+ let at;
+ while ((at = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, at).replace(/\r$/, '');
+ buffer = buffer.slice(at + 1);
+ take(line);
+ }
+ if (buffer.length > MAX_LINE_CHARS) {
+ protocolError = 'a stdout line exceeded 1 MiB';
+ stop('protocol');
+ }
+ });
+ child.stderr.on('data', (chunk) => { stderr = `${stderr}${chunk}`.slice(-MAX_STDERR_CHARS); });
+ child.on('error', (error) => { spawnError = error?.code || error?.message || 'spawn failed'; stop('spawn-error'); });
+ child.on('close', (code, sig) => {
+ closed = true;
+ clearTimeout(timer);
+ clearTimeout(killTimer);
+ ctx.signal?.removeEventListener('abort', onAbort);
+ take(`${buffer}${decoder.end()}`.trim());
+ // Whatever the CLI started in its group (MCP servers, tool shells) goes too.
+ if (OWN_GROUP && child.pid !== undefined) { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* group gone */ } }
+ trace.emit('process.exited', { code, signal: sig });
+ resolve({ state, code, stderr, ended, protocolError, spawnError });
+ });
+ child.stdin.on('error', () => {});
+ child.stdin.end(launched.stdin ?? '');
+});
+
+/**
+ * An adapter for a CLI that cannot be pointed at the run's provider. detect()
+ * still reads the pinned binary, so the matrix proves it is installed; every
+ * turn fails as RunFailure provider_unsupported with `detail`, before any
+ * process starts, so a check reads a named reason rather than a hang.
+ */
+export const createProviderUnsupportedAdapter = ({ name, cli, command, versionPattern, versionArgs, detail }) => ({ binaryPath } = {}) => {
+ const bin = binaryPath || command;
+ return {
+ name,
+ runtimeType: name,
+ supportsProvider: false,
+ async detect() { return readVersion(bin, versionPattern, { args: versionArgs || ['--version'] }); },
+ async spawn(_prompt, ctx = {}) {
+ const trace = traceOf(ctx);
+ const failure = runFailure('provider_unsupported', `${cli}: ${detail}`);
+ trace.emit('run.started', { adapter: name, cli: { name: cli }, attempt: 1 });
+ trace.emit('run.settled', { outcome: 'failed', failure: { kind: failure.kind, detail: failure.detail }, durationMs: 0, attempts: 1 });
+ throw failure;
+ },
+ };
+};
diff --git a/packages/seats/src/adapters/provider-unsupported.js b/packages/seats/src/adapters/provider-unsupported.js
new file mode 100644
index 0000000000..9016f08af0
--- /dev/null
+++ b/packages/seats/src/adapters/provider-unsupported.js
@@ -0,0 +1,36 @@
+/**
+ * CLIs on the compat matrix that cannot be pointed at the run's provider.
+ * Each detects its pinned binary like any other target, and every turn fails
+ * as RunFailure provider_unsupported naming why, so the matrix shows a named
+ * red row rather than a skip. The reasons were read from each CLI's own
+ * --help and docs at the pinned version; a release that adds a custom
+ * OpenAI- or Anthropic-compatible base URL should get a real adapter.
+ */
+import { createProviderUnsupportedAdapter } from './headless.js';
+
+export const createGeminiAdapter = createProviderUnsupportedAdapter({
+ name: 'gemini',
+ cli: 'gemini',
+ command: 'gemini',
+ versionPattern: /(\d+\.\d+\.\d+[^\s]*)/,
+ detail: 'Gemini CLI speaks only the Gemini API (GOOGLE_GEMINI_BASE_URL and Vertex both expect a Gemini endpoint); '
+ + 'it has no OpenAI- or Anthropic-compatible provider route',
+});
+
+export const createCursorAdapter = createProviderUnsupportedAdapter({
+ name: 'cursor',
+ cli: 'cursor-agent',
+ command: 'cursor-agent',
+ versionPattern: /(\d{4}\.\d+\.\d+-[0-9a-f]+)/,
+ detail: "cursor-agent sends every model call through Cursor's own backend (--endpoint names Cursor's API, not a model provider) "
+ + 'and signs in with CURSOR_API_KEY; it has no OpenAI- or Anthropic-compatible base URL',
+});
+
+export const createAmpAdapter = createProviderUnsupportedAdapter({
+ name: 'amp',
+ cli: 'amp',
+ command: 'amp',
+ versionPattern: /(\d+\.\d+\.\d+-g[0-9a-f]+)/,
+ detail: 'Amp sends every model call through the Amp service (AMP_URL names an Amp server, not a model provider) '
+ + 'and signs in with AMP_API_KEY; it has no OpenAI- or Anthropic-compatible base URL',
+});
diff --git a/packages/seats/src/adapters/qwen.js b/packages/seats/src/adapters/qwen.js
new file mode 100644
index 0000000000..373038739d
--- /dev/null
+++ b/packages/seats/src/adapters/qwen.js
@@ -0,0 +1,70 @@
+/**
+ * Qwen Code, one headless turn per process:
+ * qwen --bare --safe-mode --auth-type openai -m -o stream-json [--resume ] < prompt
+ *
+ * The same invocation the CLI's production qwen adapter uses. The provider
+ * goes in OPENAI_BASE_URL / OPENAI_API_KEY; QWEN_HOME is the run's HOME so a
+ * resume finds the session the first turn recorded.
+ *
+ * stdout is stream-json: `system/init` and every later event carry
+ * session_id; `assistant` carries text blocks; `result` closes the turn and
+ * says is_error, with the provider's text in error.message.
+ */
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHeadlessAdapter } from './headless.js';
+
+const parse = (line) => {
+ try { return JSON.parse(line); } catch { return null; }
+};
+
+export const readQwenEvent = (line, state) => {
+ const event = parse(line);
+ if (!event || typeof event !== 'object') return;
+ if (typeof event.session_id === 'string' && event.session_id) state.sessionId = event.session_id;
+ if (event.type === 'assistant') {
+ const text = (event.message?.content || [])
+ .filter((block) => block?.type === 'text' && typeof block.text === 'string')
+ .map((block) => block.text).join('');
+ if (text) state.chunks.push(text);
+ } else if (event.type === 'result') {
+ if (event.is_error) state.error = String(event.error?.message || event.result || `qwen result ${event.subtype || 'error'}`);
+ else if (typeof event.result === 'string') state.text = event.result;
+ }
+};
+
+export const createQwenAdapter = createHeadlessAdapter({
+ name: 'qwen',
+ cli: 'qwen',
+ command: 'qwen',
+ versionPattern: /(\d+\.\d+\.\d+[^\s]*)/,
+ async launch(prompt, ctx, { provider, env, home }) {
+ const qwenHome = join(home, '.qwen');
+ await mkdir(qwenHome, { recursive: true, mode: 0o700 });
+ const settingsPath = join(qwenHome, 'system-settings.json');
+ await writeFile(settingsPath, JSON.stringify({
+ privacy: { usageStatisticsEnabled: false },
+ general: { enableAutoUpdate: false },
+ }), { mode: 0o600 });
+ return {
+ args: [
+ '--bare', '--safe-mode', '--auth-type', 'openai', '-m', provider.model, '-o', 'stream-json',
+ ...(ctx.sessionId ? ['--resume', ctx.sessionId] : []),
+ ],
+ stdin: prompt,
+ env: {
+ ...env,
+ QWEN_HOME: qwenHome,
+ QWEN_RUNTIME_DIR: qwenHome,
+ QWEN_CODE_SYSTEM_SETTINGS_PATH: settingsPath,
+ QWEN_USAGE_STATISTICS_ENABLED: 'false',
+ OPENAI_BASE_URL: provider.baseURL,
+ OPENAI_API_KEY: provider.apiKey,
+ OPENAI_MODEL: provider.model,
+ },
+ };
+ },
+ read: readQwenEvent,
+});
+
+export default createQwenAdapter;
diff --git a/packages/seats/src/failures.js b/packages/seats/src/failures.js
index b3ff12fb1d..36a839b660 100644
--- a/packages/seats/src/failures.js
+++ b/packages/seats/src/failures.js
@@ -10,6 +10,7 @@ export const FAILURE_KINDS = Object.freeze({
timeout: { retryable: true, meaning: 'the run exceeded its wall-clock budget' },
exited: { retryable: true, meaning: 'the CLI process ended before the turn settled' },
protocol: { retryable: false, meaning: 'the CLI broke its wire protocol' },
+ provider_unsupported: { retryable: false, meaning: 'the CLI cannot be pointed at the run\'s OpenAI- or Anthropic-compatible provider' },
refusal: { retryable: false, meaning: 'the model or provider refused; never retried' },
max_tokens: { retryable: false, meaning: 'the turn hit its token limit' },
max_turn_requests: { retryable: false, meaning: 'the turn hit its request limit' },
diff --git a/packages/seats/test-support/fake-cli-adapter.js b/packages/seats/test-support/fake-cli-adapter.js
new file mode 100644
index 0000000000..6e136722dc
--- /dev/null
+++ b/packages/seats/test-support/fake-cli-adapter.js
@@ -0,0 +1,149 @@
+/**
+ * An adapter for test-support/fake-cli.mjs. It exists so the compatibility
+ * runner is proven against a real child process — argv, env, stdio, exit codes
+ * and kills — before any third-party CLI is wired in.
+ *
+ * `createFakeCliAdapter({ fault })` builds a deliberately broken variant, one
+ * defect per name, so each check can be shown to fail for the reason it names:
+ * adapter-side: key-in-argv, retry-refusal, no-timeout, plain-error,
+ * unscrubbed-error, bad-trace
+ * child-side (passed through FAKE_CLI_FAULT): echo-key, extra-text,
+ * new-session, error-echo-key, double-call, retry-5xx-forever
+ * version: { version } overrides what the child reports
+ */
+import { spawn as childSpawn, spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { StringDecoder } from 'node:string_decoder';
+import { RunFailure, runFailure } from '../src/failures.js';
+import { traceOf } from '../src/trace/events.js';
+import { observeSpawn } from '../src/runtime-observation.js';
+
+export const FAKE_CLI_PATH = fileURLToPath(new URL('./fake-cli.mjs', import.meta.url));
+const CHILD_FAULTS = new Set(['echo-key', 'extra-text', 'new-session', 'error-echo-key', 'double-call', 'retry-5xx-forever']);
+const DEFAULT_TIMEOUT_MS = 30000;
+const KILL_GRACE_MS = 2000;
+
+// A provider body that names a content policy is a refusal; anything else
+// from the provider is an upstream failure the CLI could not get past.
+const failureFor = (result, credentials, scrub) => {
+ const status = Number(result?.status) || 0;
+ const text = `provider answered ${status}: ${String(result?.error || '').slice(0, 300)}`;
+ const kind = status === 400 && /content.policy|content_policy/i.test(String(result?.error)) ? 'refusal' : 'exited';
+ return scrub ? runFailure(kind, text, { credentials }) : new RunFailure(kind, text);
+};
+
+const runOnce = ({ args, prompt, env, cwd, timeoutMs, signal, onSpawn, trace }) => new Promise((resolve) => {
+ const child = childSpawn(process.execPath, args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] });
+ observeSpawn(child, onSpawn, { model: env.FAKE_CLI_MODEL });
+ const decoder = new StringDecoder('utf8');
+ let buffer = '';
+ let result = null;
+ let ended = null; // why we stopped the child, if we did
+ let killTimer;
+ const stop = (reason) => {
+ if (ended) return;
+ ended = reason;
+ child.kill('SIGTERM');
+ killTimer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS);
+ };
+ const timer = timeoutMs ? setTimeout(() => stop('timeout'), timeoutMs) : null;
+ const onAbort = () => stop('aborted');
+ signal?.addEventListener('abort', onAbort, { once: true });
+ if (signal?.aborted) onAbort();
+ child.stdout.on('data', (chunk) => {
+ buffer += decoder.write(chunk);
+ let newline;
+ while ((newline = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, newline).trim();
+ buffer = buffer.slice(newline + 1);
+ if (!line) continue;
+ let event;
+ try { event = JSON.parse(line); } catch { continue; }
+ if (event.type === 'message') trace.emit('agent.message_chunk', { text: String(event.text ?? '') });
+ if (event.type === 'result') result = event;
+ }
+ });
+ child.stderr.resume();
+ child.on('error', () => stop('spawn-error'));
+ child.on('close', (code, sig) => {
+ clearTimeout(timer);
+ clearTimeout(killTimer);
+ signal?.removeEventListener('abort', onAbort);
+ trace.emit('process.exited', { code, signal: sig });
+ resolve({ result, code, ended });
+ });
+ child.stdin.on('error', () => {});
+ child.stdin.end(prompt);
+});
+
+export const createFakeCliAdapter = ({ fault = null, version = null, bin = FAKE_CLI_PATH } = {}) => ({
+ name: fault ? `fake-cli:${fault}` : 'fake-cli',
+ runtimeType: 'fake-cli',
+ providerKeyEnv: 'FAKE_CLI_API_KEY',
+
+ async detect() {
+ const env = { ...process.env, ...(version ? { FAKE_CLI_VERSION: version } : {}) };
+ const probe = spawnSync(process.execPath, [bin, '--version'], { encoding: 'utf8', env, timeout: 10000 });
+ const found = /^fake-cli (\S+)/.exec(String(probe.stdout || ''))?.[1];
+ return probe.status === 0 && found ? { path: bin, version: found } : null;
+ },
+
+ async spawn(prompt, ctx = {}) {
+ const trace = traceOf(ctx);
+ const provider = ctx.provider;
+ if (!provider?.baseURL) throw runFailure('unsupported', 'fake-cli needs ctx.provider');
+ const credentials = provider.apiKey ? [provider.apiKey] : [];
+ const env = {
+ ...(ctx.env || {}),
+ FAKE_CLI_BASE_URL: provider.baseURL,
+ FAKE_CLI_MODEL: provider.model || '',
+ FAKE_CLI_API_KEY: provider.apiKey || '',
+ ...(CHILD_FAULTS.has(fault) ? { FAKE_CLI_FAULT: fault } : {}),
+ };
+ const args = [bin, 'run'];
+ if (ctx.sessionId) args.push('--resume', ctx.sessionId);
+ if (fault === 'key-in-argv') args.push('--api-key', provider.apiKey);
+ const requested = Number(ctx.timeoutMs);
+ const timeoutMs = fault === 'no-timeout' ? 0
+ : (Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_TIMEOUT_MS);
+ const started = Date.now();
+ const settle = (outcome, failure) => {
+ const data = { outcome, durationMs: Date.now() - started, attempts };
+ if (failure) data.failure = { kind: failure.kind, detail: failure.detail };
+ trace.emit('run.settled', data);
+ };
+
+ let attempts = 0;
+ const maxAttempts = fault === 'retry-refusal' ? 2 : 1;
+ if (fault === 'bad-trace') trace.emit('turn.prompt_sent', { chars: prompt.length });
+ for (;;) {
+ attempts += 1;
+ trace.emit('run.started', { adapter: this.name, cli: { name: 'fake-cli', version: version || '1.0.0' }, sessionId: ctx.sessionId, attempt: attempts });
+ if (fault !== 'bad-trace') trace.emit('turn.prompt_sent', { chars: prompt.length });
+ const { result, ended } = await runOnce({
+ args, prompt, env, cwd: ctx.cwd, timeoutMs, signal: ctx.signal, onSpawn: ctx.onSpawn, trace,
+ });
+ let failure = null;
+ if (ended === 'aborted') {
+ const reason = ctx.signal?.reason;
+ failure = reason instanceof RunFailure ? reason : runFailure('cancelled', 'the caller cancelled the turn');
+ } else if (ended === 'timeout') {
+ failure = runFailure('timeout', `fake-cli made no answer within ${timeoutMs}ms`);
+ } else if (ended === 'spawn-error' || !result) {
+ failure = runFailure('exited', 'fake-cli ended without a result');
+ } else if (!result.ok) {
+ failure = failureFor(result, credentials, fault !== 'unscrubbed-error' && fault !== 'error-echo-key');
+ }
+ if (!failure) {
+ settle('ok');
+ return { text: String(result.text ?? ''), newSessionId: result.session_id };
+ }
+ if (fault === 'retry-refusal' && failure.kind === 'refusal' && attempts < maxAttempts) continue;
+ settle(failure.kind === 'cancelled' ? 'cancelled' : 'failed', failure);
+ if (fault === 'plain-error') throw new Error(failure.detail);
+ throw failure;
+ }
+ },
+});
+
+export default createFakeCliAdapter();
diff --git a/packages/seats/test-support/fake-cli.mjs b/packages/seats/test-support/fake-cli.mjs
new file mode 100644
index 0000000000..d9173de24e
--- /dev/null
+++ b/packages/seats/test-support/fake-cli.mjs
@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+/**
+ * A tiny agent CLI for the compatibility matrix: a real child process that
+ * reads a prompt on stdin, makes one OpenAI-compatible call, and writes JSONL.
+ *
+ * fake-cli --version prints "fake-cli "
+ * fake-cli run [--resume ID] one turn; the prompt arrives on stdin
+ *
+ * Provider settings arrive in env (FAKE_CLI_BASE_URL, FAKE_CLI_MODEL,
+ * FAKE_CLI_API_KEY), or the key as `--api-key K` when an adapter misbehaves.
+ * FAKE_CLI_FAULT selects a child-side defect so the matrix can prove it
+ * notices one: echo-key, extra-text, new-session, error-echo-key, double-call,
+ * retry-5xx-forever.
+ *
+ * Output lines: {type:'message', text} then {type:'result', ok:true, text,
+ * session_id} or {type:'result', ok:false, status, error}.
+ */
+import { randomUUID } from 'node:crypto';
+
+const VERSION = process.env.FAKE_CLI_VERSION || '1.0.0';
+const argv = process.argv.slice(2);
+const flag = (name) => {
+ const at = argv.indexOf(name);
+ return at === -1 ? null : argv[at + 1] ?? null;
+};
+const write = (event) => process.stdout.write(`${JSON.stringify(event)}\n`);
+
+if (argv[0] === '--version') {
+ process.stdout.write(`fake-cli ${VERSION}\n`);
+ process.exit(0);
+}
+if (argv[0] !== 'run') {
+ process.stderr.write('usage: fake-cli run [--resume ID]\n');
+ process.exit(2);
+}
+
+const fault = process.env.FAKE_CLI_FAULT || '';
+const key = flag('--api-key') || process.env.FAKE_CLI_API_KEY || '';
+const sessionId = (fault !== 'new-session' && flag('--resume')) || `fake-${randomUUID()}`;
+
+let prompt = '';
+process.stdin.setEncoding('utf8');
+for await (const chunk of process.stdin) prompt += chunk;
+
+const ask = () => fetch(`${process.env.FAKE_CLI_BASE_URL}/chat/completions`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+ body: JSON.stringify({ model: process.env.FAKE_CLI_MODEL, messages: [{ role: 'user', content: prompt }] }),
+}).catch((error) => ({ ok: false, status: 0, text: async () => String(error?.message || error) }));
+
+// double-call: a second request per turn, the way a CLI that also titles its
+// session behaves. retry-5xx-forever: never gives up on a server error.
+if (fault === 'double-call') await (await ask()).text();
+let response = await ask();
+while (fault === 'retry-5xx-forever' && response.status >= 500) {
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ response = await ask();
+}
+
+if (!response.ok) {
+ const body = await response.text();
+ const error = fault === 'error-echo-key' ? `provider rejected key ${key}: ${body}` : body;
+ write({ type: 'result', ok: false, status: response.status, error });
+ process.exit(1);
+}
+
+const answer = (await response.json())?.choices?.[0]?.message?.content ?? '';
+let text = answer;
+if (fault === 'echo-key') text = `${answer} (key ${key})`;
+if (fault === 'extra-text') text = `${answer}\n\nLet me know if you need anything else.`;
+write({ type: 'message', text });
+write({ type: 'result', ok: true, text, session_id: sessionId });
diff --git a/packages/seats/test-support/headless-fake-adapter.js b/packages/seats/test-support/headless-fake-adapter.js
new file mode 100644
index 0000000000..7b90898006
--- /dev/null
+++ b/packages/seats/test-support/headless-fake-adapter.js
@@ -0,0 +1,48 @@
+/**
+ * test-support/fake-cli.mjs driven through src/adapters/headless.js, so the
+ * shared one-turn runner is proven against the full compat suite with a real
+ * child process before any third-party CLI leans on it. Also an adapter for
+ * a CLI that cannot reach the provider, built the way gemini, cursor and amp
+ * are, with fake-cli as its pinned binary.
+ */
+import { FAKE_CLI_PATH } from './fake-cli-adapter.js';
+import { createHeadlessAdapter, createProviderUnsupportedAdapter } from '../src/adapters/headless.js';
+
+const versionArgs = [FAKE_CLI_PATH, '--version'];
+
+export const readFakeCliLine = (line, state) => {
+ const event = JSON.parse(line);
+ if (event.type === 'message' && event.text) state.chunks.push(event.text);
+ if (event.type !== 'result') return;
+ if (event.ok) {
+ state.text = event.text;
+ state.sessionId = event.session_id;
+ } else {
+ state.error = `${event.status}: ${event.error}`;
+ }
+};
+
+export const createHeadlessFakeAdapter = createHeadlessAdapter({
+ name: 'headless-fake',
+ cli: 'fake-cli',
+ command: process.execPath,
+ versionArgs,
+ versionPattern: /fake-cli (\S+)/,
+ async launch(prompt, ctx, { provider, env }) {
+ return {
+ args: [FAKE_CLI_PATH, 'run', ...(ctx.sessionId ? ['--resume', ctx.sessionId] : [])],
+ stdin: prompt,
+ env: { ...env, FAKE_CLI_BASE_URL: provider.baseURL, FAKE_CLI_MODEL: provider.model, FAKE_CLI_API_KEY: provider.apiKey },
+ };
+ },
+ read: readFakeCliLine,
+});
+
+export const createUnsupportedFakeAdapter = createProviderUnsupportedAdapter({
+ name: 'unsupported-fake',
+ cli: 'fake-cli',
+ command: process.execPath,
+ versionArgs,
+ versionPattern: /fake-cli (\S+)/,
+ detail: 'this CLI only talks to its vendor\'s own service',
+});
diff --git a/packages/seats/test/compat-install.test.js b/packages/seats/test/compat-install.test.js
new file mode 100644
index 0000000000..5fddea9453
--- /dev/null
+++ b/packages/seats/test/compat-install.test.js
@@ -0,0 +1,96 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { installTarget } from '../compat/install.js';
+import { resolveBinary } from '../compat/targets.js';
+
+const withTmp = async (fn) => {
+ const dir = mkdtempSync(join(tmpdir(), 'seats-compat-install-'));
+ try { return await fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); }
+};
+
+// A release tarball holding one executable, served from memory.
+const tarball = (dir) => {
+ const src = join(dir, 'src');
+ mkdirSync(src);
+ writeFileSync(join(src, 'tool'), '#!/bin/sh\necho tool 2.0.0\n');
+ const archive = join(dir, 'tool.tar.gz');
+ assert.equal(spawnSync('tar', ['-czf', archive, '-C', src, 'tool']).status, 0);
+ const body = readFileSync(archive);
+ return { body, sha256: createHash('sha256').update(body).digest('hex') };
+};
+const serve = (body) => async () => ({ ok: true, status: 200, arrayBuffer: async () => body });
+const target = (asset) => ({
+ cli: 'tool', version: '2.0.0', adapter: 'src/adapters/stub.js#default', enabled: true,
+ binary: { bin: 'tool', release: { 'linux-x64': { url: 'https://example.invalid/tool.tar.gz', ...asset } } },
+});
+
+test('a release asset is installed into /- only when its sha256 matches', () => withTmp(async (dir) => {
+ const { body, sha256 } = tarball(dir);
+ const prefix = join(dir, 'prefix');
+ const ok = await installTarget(target({ sha256 }), { prefix, platform: 'linux-x64', fetchImpl: serve(body) });
+ assert.equal(ok.installed, join(prefix, 'tool-2.0.0', 'tool'));
+ assert.ok(existsSync(ok.installed));
+
+ const other = join(dir, 'other');
+ await assert.rejects(
+ installTarget(target({ sha256: 'a'.repeat(64) }), { prefix: other, platform: 'linux-x64', fetchImpl: serve(body) }),
+ /sha256 mismatch/,
+ );
+ assert.equal(existsSync(join(other, 'tool-2.0.0', 'tool')), false, 'nothing is unpacked on a mismatch');
+ await assert.rejects(
+ installTarget(target({}), { prefix: other, platform: 'linux-x64', fetchImpl: serve(body) }),
+ /no pinned sha256/,
+ );
+}));
+
+test('install skips what the runner would skip', () => withTmp(async (dir) => {
+ const absent = { ...target({ sha256: 'a'.repeat(64) }), adapter: 'src/adapters/not-landed.js#x' };
+ assert.match((await installTarget(absent, { prefix: dir, platform: 'linux-x64' })).skipped, /not present/);
+ assert.match((await installTarget(target({ sha256: 'a'.repeat(64) }), { prefix: dir, platform: 'win32-x64' })).skipped, /win32-x64/);
+ assert.equal((await installTarget({ ...target({}), enabled: false }, { prefix: dir })).skipped, 'disabled');
+}));
+
+test('binary paths resolve per platform and never put the host home in the label', () => {
+ const npm = { cli: 'oc', version: '1.0.0', binary: { npm: 'oc@1.0.0', bin: { 'linux-x64': 'node_modules/oc-linux-x64/bin/oc' } } };
+ const resolved = resolveBinary(npm, { prefix: '/opt/compat', platform: 'linux-x64' });
+ assert.equal(resolved.path, '/opt/compat/oc-1.0.0/node_modules/oc-linux-x64/bin/oc');
+ assert.equal(resolved.label, '/oc-1.0.0/node_modules/oc-linux-x64/bin/oc');
+ assert.match(resolveBinary(npm, { prefix: '/opt', platform: 'darwin-x64' }).missing, /darwin-x64/);
+ assert.match(resolveBinary(npm, { platform: 'linux-x64' }).missing, /prefix/);
+});
+
+test('a pip spec is pinned exactly and installed into /venv from wheels only', () => withTmp(async (dir) => {
+ const calls = [];
+ const runImpl = (command, args) => {
+ calls.push([command, ...args]);
+ // Stand in for the venv and the console script pip would create.
+ if (args[0] === 'install') {
+ mkdirSync(join(dir, 'prefix', 'tool-2.0.0', 'venv', 'bin'), { recursive: true });
+ writeFileSync(join(dir, 'prefix', 'tool-2.0.0', 'venv', 'bin', 'tool'), '#!/bin/sh\n');
+ }
+ };
+ const pip = {
+ cli: 'tool', version: '2.0.0', adapter: 'src/adapters/stub.js#default', enabled: true,
+ binary: { pip: 'tool-cli==2.0.0', bin: 'venv/bin/tool' },
+ };
+ const prefix = join(dir, 'prefix');
+ const ok = await installTarget(pip, { prefix, platform: 'linux-x64', pipOptions: { python: 'python3.12', runImpl } });
+ assert.equal(ok.installed, join(prefix, 'tool-2.0.0', 'venv', 'bin', 'tool'));
+ const venv = join(prefix, 'tool-2.0.0', 'venv');
+ assert.deepEqual(calls[0], ['python3.12', '-m', 'venv', venv]);
+ assert.equal(calls[1][0], join(venv, 'bin', 'pip'));
+ assert.ok(calls[1].includes('--only-binary=:all:'), 'no sdist, so no setup.py runs');
+ assert.equal(calls[1].at(-1), 'tool-cli==2.0.0');
+
+ for (const loose of ['tool-cli', 'tool-cli>=2', 'tool-cli==2.0.0; rm -rf /', '-r requirements.txt']) {
+ await assert.rejects(
+ installTarget({ ...pip, binary: { ...pip.binary, pip: loose } }, { prefix: join(dir, 'p2'), platform: 'linux-x64', pipOptions: { runImpl } }),
+ /must pin one exact version/,
+ );
+ }
+}));
diff --git a/packages/seats/test/compat-probe.test.js b/packages/seats/test/compat-probe.test.js
new file mode 100644
index 0000000000..6b31f6e8b0
--- /dev/null
+++ b/packages/seats/test/compat-probe.test.js
@@ -0,0 +1,157 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { CHECKS, readRetryProbe, runCheck } from '../compat/checks.js';
+import { tapSpawn } from '../compat/harness.js';
+
+const KEY = ['sk', 'compat', 'probe', 'key', '0001'].join('-');
+const check = (id) => CHECKS.find((c) => c.id === id);
+const TIMEOUTS = { turnMs: 2000, stallMs: 500, graceMs: 200, checkMs: 3000, retryWindowMs: 500 };
+
+// `requests` is the harness stamp: provider requests made by that instant.
+const update = (requests, sessionUpdate, extra = {}) => ({
+ dir: 'in', ms: requests * 100, requests, msg: { jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'ses_1', update: { sessionUpdate, ...extra } } },
+});
+const prompt = { dir: 'out', ms: 10, requests: 0, msg: { jsonrpc: '2.0', id: 3, method: 'session/prompt', params: { sessionId: 'ses_1', prompt: [] } } };
+const retryMeta = { 'opencode/retry': { attempt: 1, nextRetryAt: 1761390000000, error: { type: 'api', message: 'fake provider failure', status: 503 } } };
+const fails = (n) => Array.from({ length: n }, () => ({ directive: 'fail' }));
+
+// 1.19.4 as measured in a container run: available_commands_update can land
+// just after session/prompt is written, but before any provider request.
+const silentTurn = () => ({
+ ok: false, ms: 500, tapped: true, requests: fails(4),
+ error: Object.assign(new Error('timeout'), { name: 'RunFailure', kind: 'timeout' }),
+ events: [{ type: 'session.opened', requests: 0 }, { type: 'turn.prompt_sent', requests: 0 }, { type: 'process.exited', requests: 4 }],
+ wire: [prompt, update(0, 'available_commands_update')],
+});
+// The shape PR #50752 describes: a session_info_update carrying only _meta.
+const reportingTurn = ({ surfaced = [] } = {}) => ({
+ ...silentTurn(),
+ events: [{ type: 'session.opened', requests: 0 }, { type: 'turn.prompt_sent', requests: 0 },
+ ...surfaced.map(([type, requests]) => ({ type, requests })), { type: 'process.exited', requests: 4 }],
+ wire: [prompt, update(0, 'available_commands_update'),
+ update(2, 'session_info_update', { _meta: retryMeta }),
+ update(3, 'session_info_update', { _meta: { 'opencode/retry': null } }),
+ // After the last request: what an agent says once it has given up.
+ update(4, 'agent_message_chunk', { content: { type: 'text', text: 'Ran into this error' } }),
+ { dir: 'in', ms: 3000, requests: 4, msg: { jsonrpc: '2.0', id: 3, result: { stopReason: 'cancelled', _meta: retryMeta } } }],
+});
+
+const envWith = (turn) => ({
+ timeouts: TIMEOUTS,
+ provider: { apiKey: KEY },
+ detect: async () => ({ found: { path: '/bin/x', version: '1' } }),
+ probeTurn: turn === undefined ? undefined : () => Promise.resolve(turn),
+});
+
+test('readRetryProbe counts only updates between the first and the last provider request', () => {
+ const silent = readRetryProbe(silentTurn());
+ assert.equal(silent.updates, 0, 'an update before any provider request is not retry progress');
+ assert.deepEqual(silent.otherKinds, { available_commands_update: 1 });
+ assert.equal(silent.failedRequests, 4);
+ const reporting = readRetryProbe(reportingTurn(), { key: KEY });
+ assert.equal(reporting.updates, 2);
+ assert.deepEqual(reporting.kinds, { session_info_update: 2 });
+ assert.deepEqual(reporting.otherKinds, { available_commands_update: 1, agent_message_chunk: 1 }, 'the give-up text is not progress');
+ assert.deepEqual(reporting.metaKeys, ['opencode/retry']);
+ assert.deepEqual(reporting.answerMetaKeys, ['opencode/retry']);
+ assert.deepEqual(reporting.surfaced, []);
+ assert.match(reporting.sample, /"sessionUpdate":"session_info_update"/);
+});
+
+test('readRetryProbe scrubs the key from the sample it quotes', () => {
+ const turn = reportingTurn();
+ turn.wire[2] = update(2, 'session_info_update', { _meta: { echo: KEY } });
+ const probe = readRetryProbe(turn, { key: KEY });
+ assert.doesNotMatch(probe.sample, new RegExp(KEY));
+ assert.match(probe.sample, //);
+});
+
+test('retry-status fails a silent retry and passes one the agent reports', async () => {
+ const silent = await runCheck(check('retry-status'), {}, envWith(silentTurn()));
+ assert.equal(silent.status, 'fail');
+ assert.match(silent.detail, /4 provider requests in 500ms and no session\/update between the first and the last; outside the retries: available_commands_update×1/);
+ const reporting = await runCheck(check('retry-status'), {}, envWith(reportingTurn()));
+ assert.equal(reporting.status, 'pass', reporting.detail);
+ assert.match(reporting.detail, /2 session\/update\(s\) between the first and the last of 4 provider requests in 500ms: session_info_update×2; _meta keys opencode\/retry; outside the retries: available_commands_update×1, agent_message_chunk×1; the prompt answer carries _meta opencode\/retry/);
+});
+
+test('retry-status passes an agent that does not retry, and skips when it cannot see the agent', async () => {
+ const once = { ...silentTurn(), requests: fails(1) };
+ assert.equal((await runCheck(check('retry-status'), {}, envWith(once))).status, 'pass');
+ const noProbe = await runCheck(check('retry-status'), {}, envWith(undefined));
+ assert.equal(noProbe.status, 'skip');
+ assert.match(noProbe.detail, /declares no probeOptions/);
+ const untapped = await runCheck(check('retry-status'), {}, envWith({ ...silentTurn(), tapped: false, wire: [] }));
+ assert.equal(untapped.status, 'skip');
+ assert.match(untapped.detail, /took no spawn hook/);
+ const noPrompt = await runCheck(check('retry-status'), {}, envWith({ ...silentTurn(), wire: [] }));
+ assert.equal(noPrompt.status, 'skip');
+});
+
+test('retry-status-surfaced fails when the trace drops what the agent reported', async () => {
+ const dropped = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn()));
+ assert.equal(dropped.status, 'fail');
+ assert.match(dropped.detail, /the agent sent session_info_update×2 \(_meta keys opencode\/retry\); the trace carries no event from them/);
+ const carried = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn({ surfaced: [['agent.thought_chunk', 2], ['agent.thought_chunk', 3]] })));
+ assert.equal(carried.status, 'pass', carried.detail);
+ // An event from the give-up text does not stand in for a dropped retry report.
+ const late = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn({ surfaced: [['agent.message_chunk', 4], ['usage', 4]] })));
+ assert.equal(late.status, 'fail', late.detail);
+ const nothing = await runCheck(check('retry-status-surfaced'), {}, envWith(silentTurn()));
+ assert.equal(nothing.status, 'skip');
+});
+
+test('tapSpawn records JSON lines both ways, split across chunks, and drops the rest', async () => {
+ const wire = [];
+ const script = [
+ "process.stdin.on('data', (d) => {",
+ " process.stdout.write('{\"jsonrpc\":\"2.0\",\"id\":1,');",
+ " setTimeout(() => { process.stdout.write('\"result\":{}}\\nnot json\\n'); process.exit(0); }, 20);",
+ '});',
+ ].join('\n');
+ let n = 0;
+ const child = tapSpawn(wire, undefined, { stamp: () => ({ requests: n++ }) })(process.execPath, ['-e', script], { stdio: ['pipe', 'pipe', 'pipe'] });
+ child.stdout.on('data', () => {}); // the adapter's own reader
+ child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize' })}\n`);
+ await new Promise((resolve) => child.once('close', resolve));
+ assert.deepEqual(wire.map((w) => [w.dir, w.msg.method ?? 'answer']), [['out', 'initialize'], ['in', 'answer']]);
+ assert.ok(wire.every((w) => Number.isFinite(w.ms)));
+ assert.deepEqual(wire.map((w) => w.requests), [0, 1], 'each record carries the stamp');
+});
+
+// goose 1.56.0 as measured: a run marker before any request, two session-state
+// updates beside the first, then nothing until its error text after the last.
+test('start-of-turn bookkeeping and session state are not retry progress', async () => {
+ const turn = {
+ ...silentTurn(),
+ requests: fails(8),
+ events: [{ type: 'turn.prompt_sent', requests: 0 }, { type: 'usage', requests: 1 }, { type: 'usage', requests: 3 }, { type: 'agent.message_chunk', requests: 8 }],
+ wire: [prompt,
+ update(0, 'session_info_update', { _meta: { goose: { activeRunId: 'run_1' } } }),
+ update(1, 'usage_update', { used: 0, size: 128000 }),
+ update(1, 'available_commands_update', { availableCommands: [] }),
+ update(3, 'usage_update', { used: 0, size: 128000 }),
+ update(8, 'agent_message_chunk', { content: { type: 'text', text: 'Ran into this error: 503' } }),
+ update(8, 'session_info_update', { _meta: { goose: { activeRunId: null } } })],
+ };
+ const probe = readRetryProbe(turn);
+ assert.equal(probe.updates, 0, JSON.stringify(probe));
+ assert.deepEqual(probe.surfaced, [], 'a usage event is session state, not a surfaced retry report');
+ const result = await runCheck(check('retry-status'), {}, envWith(turn));
+ assert.equal(result.status, 'fail');
+ assert.match(result.detail, /8 provider requests .* no session\/update between the first and the last; outside the retries: session_info_update×2, usage_update×2, available_commands_update×1, agent_message_chunk×1/);
+});
+
+// While only the first request is out nothing has been retried yet, so what
+// the agent streams then (a thought, a run marker) is not a retry report.
+test('an update while only the first request is out is not retry progress', () => {
+ const turn = {
+ ...silentTurn(),
+ wire: [prompt,
+ update(1, 'agent_thought_chunk', { content: { type: 'text', text: 'thinking' } }),
+ update(1, 'session_info_update', { _meta: { goose: { activeRunId: 'run_1' } } })],
+ };
+ const probe = readRetryProbe(turn);
+ assert.equal(probe.updates, 0, JSON.stringify(probe));
+ assert.deepEqual(probe.otherKinds, { agent_thought_chunk: 1, session_info_update: 1 });
+});
diff --git a/packages/seats/test/compat-render.test.js b/packages/seats/test/compat-render.test.js
new file mode 100644
index 0000000000..2ecbfc8b8b
--- /dev/null
+++ b/packages/seats/test/compat-render.test.js
@@ -0,0 +1,83 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { escapeHtml, limitText, renderMatrix } from '../compat/render.js';
+
+const results = (detail) => ({
+ schema: 'seats-compat/1',
+ generatedAt: '2026-10-25T00:00:00.000Z',
+ host: { os: 'linux 6.1', platform: 'linux', arch: 'x64', node: 'v22.0.0' },
+ checks: [{ id: 'detect', title: 'detect() finds binary' }, { id: 'refusal', title: 'refusal' }],
+ targets: [{ target: 'evil@1">
', cli: 'evil', version: '1', note: 'note' }],
+ rows: [
+ { target: 'evil@1">
', check: 'detect', status: 'pass', detail: 'ok', ms: 3 },
+ { target: 'evil@1">
', check: 'refusal', status: 'fail', detail, ms: 9 },
+ ],
+});
+
+test('the renderer escapes a & more'));
+ assert.doesNotMatch(html, /