diff --git a/.github/workflows/seats-compat-nightly.yml b/.github/workflows/seats-compat-nightly.yml new file mode 100644 index 0000000000..f9e38f6bf6 --- /dev/null +++ b/.github/workflows/seats-compat-nightly.yml @@ -0,0 +1,89 @@ +name: Seats compatibility nightly + +# Runs each pinned agent CLI (packages/seats/compat/targets.json) through the +# compatibility suite against a local fake provider: no model provider, no +# real key. A failing check is a result, not a job failure; the job fails +# only when the matrix itself could not be produced. + +on: + schedule: + - cron: '23 9 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: seats-compat-nightly + cancel-in-progress: false + +jobs: + matrix: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + # The runner context is not available in job-level env, so the + # job-local paths are set here for every later step. + - name: Choose job-local paths + run: | + echo "COMPAT_PREFIX=$RUNNER_TEMP/compat-bin" >> "$GITHUB_ENV" + echo "COMPAT_OUT=$RUNNER_TEMP/compat-out" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + # Aider (aider-chat) supports python 3.10 to 3.12 only. + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + # Job-local prefix only: release assets are sha256-checked before they + # are unpacked, npm packages install with --ignore-scripts, pip packages + # install from wheels into a venv under the prefix. A CLI that + # fails to install shows up in the matrix as "binary not installed". + - name: Install pinned CLIs + continue-on-error: true + run: node packages/seats/compat/install.js --targets packages/seats/compat/targets.json --prefix "$COMPAT_PREFIX" + + - name: Run the compatibility matrix + run: node packages/seats/compat/run.js --targets packages/seats/compat/targets.json --prefix "$COMPAT_PREFIX" --out "$COMPAT_OUT" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: seats-compat + path: ${{ env.COMPAT_OUT }} + if-no-files-found: error + retention-days: 30 + + publish: + # Disabled on purpose: publishing needs a maintainer to enable GitHub Pages + # for this repository (Settings → Pages → Source: GitHub Actions). Flip this + # to `if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'` + # once Pages is on. + if: false + needs: matrix + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - uses: actions/download-artifact@v4 + with: + name: seats-compat + path: site + - uses: actions/upload-pages-artifact@v3 + with: + path: site + - id: deploy + uses: actions/deploy-pages@v4 diff --git a/packages/seats/README.md b/packages/seats/README.md index 522df333f3..81c25d1ce8 100644 --- a/packages/seats/README.md +++ b/packages/seats/README.md @@ -24,3 +24,24 @@ It is private and is not published on its own. `@commonlyai/cli` depends on it a The extraction plan is D62, extract-in-place. **Run record.** The trace store writes each run as a versioned, hash-chained JSONL record. It covers who granted which agent what access for which task, who approved each call, what was attempted, and the result. A tool outside this package can read, verify and export it to OTLP. The format is specified in [`docs/record-v1.md`](docs/record-v1.md), and a generated sample is in [`docs/examples/`](docs/examples/). + +## Compatibility matrix + +`compat/` runs each pinned CLI in `compat/targets.json` through a suite of checks (`compat/checks.js`) against the local fake provider in `test-support/`, and writes `results.json` plus a self-contained `index.html`: + +``` +node compat/install.js --targets compat/targets.json --prefix # pinned binaries, job-local +node compat/run.js --targets compat/targets.json --prefix --out +``` + +A target is data: `{ cli, version, adapter: 'module#export', binary?, enabled, requires?, probeOptions?, limits? }`. The export is an adapter or a factory taking `options`. A target whose adapter module is absent skips every check. `.github/workflows/seats-compat-nightly.yml` runs it nightly and uploads the output. + +- `probeOptions` builds a second instance of a factory export for the checks that read the agent itself. The ACP targets pass `{ watchProvider: false }`, so `retry-status` sees what the agent streams while it retries a 5xx, with no provider watch ending the turn first. `retry-status-surfaced` then asks whether the adapter's trace carried those reports. +- `binary` names one install recipe: `release` (a per-platform tarball with its sha256), `npm` (installed with `--ignore-scripts`), or `pip` (an exact `name==version`, installed from wheels into a venv under the prefix; the interpreter is `python3` or `COMPAT_PYTHON`). +- CLIs that run one turn per process in their own headless mode (copilot, qwen-code, cline, aider) share `src/adapters/headless.js`: a `launch` that returns argv and env, a `read` per stdout line, and the process group, deadline, cancellation and trace done once. +- A CLI that cannot be pointed at the run's OpenAI- or Anthropic-compatible provider (gemini, cursor, amp) still gets a row. Its adapter (`src/adapters/provider-unsupported.js`) detects the pinned binary and fails every turn as `provider_unsupported` with the reason, and every turn check reports that failure, never a skip. +- `limits: { : { note, fixedIn? } }` marks a failure that belongs to that CLI version, not to the adapter. The row keeps `status: 'fail'` and gains `limit`; the page shows it as a limit with the version that fixes it, and `totals.fail` counts only failures no limit explains. A declared limit that stops reproducing shows as a pass that says so. + +### Upstream bugs + +`compat/upstream.json` lists each known upstream bug with what we do about it: `{ id, cli, versions, bug, status, workaround: { name, ref, where, tests[] }, matrix?: { target, check }, references[] }`. We do not wait on upstream, so `status` is `"worked around"` or `"open"`, and `compat/upstream.js` enforces the first: a row reads as worked around only when its workaround names the pushed commit it lives on (`ref`, a full sha) and at least one test there that proves it. A row that claims more is shown as open, with the reason. An open row names its planned workaround. Upstream issue links go in `references`, never in place of a workaround. A row with `matrix` shows what that check did on this run beside the claim. `run.js` reads the file next to the targets, or the one `--upstream` names. diff --git a/packages/seats/compat/checks.js b/packages/seats/compat/checks.js new file mode 100644 index 0000000000..92e5060cb2 --- /dev/null +++ b/packages/seats/compat/checks.js @@ -0,0 +1,365 @@ +/** + * The compatibility suite. Each check is + * { id, title, run(target, env) → { status: 'pass'|'fail'|'skip', detail, ms? } } + * where `env` is the per-target harness (compat/harness.js). Checks share turns + * through env.turn(name, …) so a fresh PONG is paid for once, not five times. + * + * Every check runs under its own timeout (runCheck); a check that never + * settles is reported as a failure, and its open turns are cancelled. + */ +import { isNamedFailure } from './harness.js'; + +const pass = (detail) => ({ status: 'pass', detail }); +const fail = (detail) => ({ status: 'fail', detail }); +const skip = (detail) => ({ status: 'skip', detail }); + +const PONG = 'PONG'; +const RESUMED = 'PONG-2'; + +// Prompts carry a fake-provider directive, so a real CLI's own system prompt +// cannot change what the provider answers. +const freshTurn = (env) => env.turn('fresh', `Reply with the single word ${PONG}. [[reply:${PONG}]]`); +const resumedTurn = async (env) => { + const first = await freshTurn(env); + const sessionId = first.ok ? first.result.newSessionId : null; + if (!sessionId) return null; + return env.turn('resumed', `Reply with the single word ${RESUMED}. [[reply:${RESUMED}]]`, { sessionId }); +}; +const refusalTurn = (env) => env.turn('refusal', 'Answer this. [[refuse]]'); +// A provider watch that stops the CLI at the first refusal on the wire names +// it refusal_retry_loop; both kinds are final and never retried. +const REFUSAL_KINDS = new Set(['refusal', 'refusal_retry_loop']); +const upstreamTurn = (env) => env.turn('upstream-5xx', 'Answer this. [[fail:503]]'); +const stallTurn = (env) => env.turn('stall', 'Answer this. [[stall]]', { timeoutMs: env.timeouts.stallMs }); +const ALL_TURNS = [freshTurn, resumedTurn, refusalTurn, upstreamTurn, stallTurn]; +// The same 503, on the probe adapter (the target's probeOptions, e.g. no +// provider watch), held open for a window so the agent's own retries happen. +const retryProbeTurn = (env) => env.probeTurn?.('retry-5xx', 'Answer this. [[fail:503]]', { timeoutMs: env.timeouts.retryWindowMs }) ?? null; + +// Trace events an adapter emits on its own account rather than from what the +// agent streamed; they say nothing about whether a retry report got through. +const LIFECYCLE_EVENT = /^(?:run\.|session\.|turn\.|process\.|permission\.|liveness\.|credential\.)/; +// ACP update kinds that publish session state (commands, mode, config, +// context size). Agents send them whenever they like, and goose 1.56.0 sends +// two right as its first request goes out, so they never count as a report +// about a retry; `usage` is what the adapter maps usage_update to. +const SESSION_STATE = new Set(['available_commands_update', 'current_mode_update', 'config_option_update', 'usage_update']); +const SESSION_STATE_EVENT = new Set(['usage']); +const SAMPLE_CHARS = 600; + +const tally = (records) => { + const kinds = {}; + const metaKeys = new Set(); + for (const { msg } of records) { + const update = msg.params?.update || {}; + const kind = update.sessionUpdate || 'unknown'; + kinds[kind] = (kinds[kind] || 0) + 1; + for (const name of Object.keys(update._meta || {})) metaKeys.add(name); + } + return { kinds, metaKeys: [...metaKeys].sort() }; +}; + +/** + * What a recorded ACP turn shows about retries: how often the provider was + * asked, what the agent streamed while it was still retrying, and what of + * that reached the trace. Reads only the tap, so it holds for any ACP agent. + * + * "While retrying" is placed by the harness stamp (provider requests made so + * far), not inferred from a kind name: an update counts when the agent had + * already asked the provider twice and asked again afterwards. The first + * request is the turn starting, and start-of-turn bookkeeping lands beside + * it; what an agent sends once it has given up (its error text) lands after + * the last. Both are reported, as "outside the retries", never as progress. + */ +export const readRetryProbe = (turn, { key = '' } = {}) => { + const wire = Array.isArray(turn?.wire) ? turn.wire : []; + const prompt = wire.find((w) => w.dir === 'out' && w.msg?.method === 'session/prompt'); + const at = prompt ? wire.indexOf(prompt) : -1; + const after = at === -1 ? [] : wire.slice(at + 1); + const total = (turn?.requests || []).length; + const inWindow = (record) => Number.isFinite(record.requests) && record.requests >= 2 && record.requests < total; + const all = after.filter((w) => w.dir === 'in' && w.msg?.method === 'session/update'); + const during = all.filter((w) => inWindow(w) && !SESSION_STATE.has(w.msg.params?.update?.sessionUpdate)); + const answer = prompt ? after.find((w) => w.dir === 'in' && w.msg?.id === prompt.msg.id && !w.msg.method) : null; + const { kinds, metaKeys } = tally(during); + const rest = tally(all.filter((w) => !during.includes(w))); + const promptEvent = (turn?.events || []).findIndex((e) => e.type === 'turn.prompt_sent'); + const surfaced = promptEvent === -1 ? [] + : turn.events.slice(promptEvent + 1).filter((e) => !LIFECYCLE_EVENT.test(e.type) && !SESSION_STATE_EVENT.has(e.type) && inWindow(e)); + const scrub = (text) => (key ? text.split(key).join('') : text); + return { + tapped: !!turn?.tapped, + promptSent: !!prompt, + failedRequests: (turn?.requests || []).filter((r) => r.directive === 'fail').length, + updates: during.length, + kinds, + metaKeys, + otherKinds: rest.kinds, + sample: during[0] ? scrub(JSON.stringify(during[0].msg.params)).slice(0, SAMPLE_CHARS) : null, + answerMetaKeys: Object.keys(answer?.msg?.result?._meta || answer?.msg?.error?.data?._meta || {}).sort(), + surfaced: surfaced.map((e) => e.type), + }; +}; + +const listKinds = (kinds) => Object.entries(kinds).map(([kind, n]) => `${kind}×${n}`).join(', '); +const needsProbe = async (env, then) => needsBinary(env, async () => { + const turn = await retryProbeTurn(env); + if (!turn) return skip('the target declares no probeOptions, so the agent is only seen through its adapter'); + const probe = readRetryProbe(turn, { key: env.provider.apiKey }); + if (!probe.tapped) return skip('the adapter took no spawn hook, so the agent\'s own stream was not recorded'); + if (!probe.promptSent) return skip(`no session/prompt was sent (${turn.ok ? 'the turn succeeded' : describeError(turn.error)})`); + return then(turn, probe); +}); + +const describeError = (error) => (isNamedFailure(error) + ? `RunFailure ${error.kind}: ${error.detail || error.message}` + : `unnamed ${error?.name || typeof error}: ${String(error?.message ?? error)}`); +const quote = (text, limit = 160) => JSON.stringify(String(text ?? '').slice(0, limit)); + +// The turn checks need a binary; without one they would only repeat detect's +// failure in five more confusing words. They also need a CLI that can reach +// the run's provider: one that cannot fails every turn check with its named +// reason, because a refusal, a 5xx or a stall it never saw proves nothing, +// and provider-5xx or stall would otherwise pass on the instant failure. +const needsBinary = async (env, then) => { + const { found } = await env.detect(); + if (!found) return skip('detect() found no binary'); + if (env.adapter?.supportsProvider === false) { + const turn = await freshTurn(env); + return fail(turn.ok + ? 'the adapter declares it cannot reach the provider, yet a turn succeeded' + : `cannot reach the run's provider: ${describeError(turn.error)}`); + } + return then(); +}; + +export const CHECKS = [ + { + id: 'detect', + title: 'detect() finds the binary and reports a version', + async run(target, env) { + const { found, error } = await env.detect(); + if (error) return fail(`detect() threw: ${describeError(error)}`); + if (!found) return fail(env.binaryPath ? `detect() returned null (expected ${target.binaryLabel || 'the pinned binary'})` : 'detect() returned null'); + if (!found.version) return fail('detect() returned no version'); + return pass(`found ${found.version}`); + }, + }, + { + id: 'version-pin', + title: "the version matches the target's pin", + async run(target, env) { + const { found } = await env.detect(); + if (!found?.version) return skip('no version to compare'); + return String(found.version) === String(target.version) + ? pass(`pinned ${target.version}`) + : fail(`detected ${found.version}, pinned ${target.version}`); + }, + }, + { + id: 'fresh-turn', + title: 'one fresh turn answers PONG through the fake provider', + run: (target, env) => needsBinary(env, async () => { + const turn = await freshTurn(env); + if (!turn.ok) return fail(`turn failed: ${describeError(turn.error)}`); + if (!turn.requests.length) return fail('the provider was never called'); + const text = String(turn.result.text ?? ''); + return text.includes(PONG) + ? pass(`answered in ${turn.ms}ms with ${turn.requests.length} provider request(s)`) + : fail(`reply ${quote(text)} does not contain ${PONG}`); + }), + }, + { + id: 'exact-reply', + title: 'the reply text is exact, with nothing extra', + run: (target, env) => needsBinary(env, async () => { + const turn = await freshTurn(env); + if (!turn.ok) return skip('no reply (fresh turn failed)'); + const text = String(turn.result.text ?? ''); + if (text === PONG) return pass(`reply is exactly ${PONG}`); + return text.trim() === PONG + ? pass(`reply is ${PONG} with surrounding whitespace ${quote(text)}`) + : fail(`reply ${quote(text)} is not exactly ${PONG}`); + }), + }, + { + id: 'resume-session', + title: 'a resumed second turn keeps the session id', + run: (target, env) => needsBinary(env, async () => { + const first = await freshTurn(env); + if (!first.ok) return skip('no first turn to resume'); + const id = first.result.newSessionId; + if (!id) return skip('the adapter returns no session id'); + const second = await resumedTurn(env); + if (!second.ok) return fail(`resumed turn failed: ${describeError(second.error)}`); + const text = String(second.result.text ?? ''); + if (!text.includes(RESUMED)) return fail(`resumed turn answered ${quote(text)}, expected ${RESUMED}`); + const next = second.result.newSessionId; + if (next && next !== id) return fail('resumed turn returned a different session id'); + return pass(next ? 'session id kept across turns' : 'resumed; the adapter returned no new id, so the first is kept'); + }), + }, + { + id: 'refusal', + title: "a provider refusal surfaces as RunFailure kind 'refusal' (or 'refusal_retry_loop', stopped on the wire) and is not retried", + run: (target, env) => needsBinary(env, async () => { + const turn = await refusalTurn(env); + // Real CLIs make more than one provider call per turn (opencode also + // asks for a session title), so "retried" means more calls than an + // ordinary turn makes, measured on this target's own fresh turn. + const fresh = await freshTurn(env); + const baseline = fresh.ok && fresh.requests.length ? fresh.requests.length : 1; + const asked = turn.requests.filter((r) => r.directive === 'refuse').length; + if (turn.ok) return fail(`a refused turn reported success: ${quote(turn.result.text)}`); + if (!isNamedFailure(turn.error) || !REFUSAL_KINDS.has(turn.error.kind)) return fail(`expected RunFailure refusal or refusal_retry_loop, got ${describeError(turn.error)}`); + if (asked === 0) return fail('reported refusal without asking the provider'); + if (asked > baseline) return fail(`the provider was asked ${asked} times against ${baseline} on an ordinary turn; a refusal must not be retried`); + return pass(`${turn.error.kind} named after ${asked} provider request(s), as on an ordinary turn`); + }), + }, + { + id: 'provider-5xx', + title: 'a provider 5xx surfaces as a named failure, not a hang', + run: (target, env) => needsBinary(env, async () => { + const turn = await upstreamTurn(env); + if (turn.ok) return fail(`a 503 turn reported success: ${quote(turn.result.text)}`); + if (!isNamedFailure(turn.error)) return fail(describeError(turn.error)); + // Ending only at the turn's own deadline means the 5xx was never + // surfaced: the CLI kept retrying until the adapter gave up on it. + if (['timeout', 'stuck'].includes(turn.error.kind)) return fail(`the 503 was not surfaced; the turn ran until RunFailure ${turn.error.kind} after ${turn.ms}ms`); + return pass(`RunFailure ${turn.error.kind} after ${turn.ms}ms`); + }), + }, + { + id: 'retry-status', + title: 'while it retries a provider 5xx, the agent reports progress on its own stream', + timeoutMs: (env) => env.timeouts.retryWindowMs + env.timeouts.graceMs * 3, + run: (target, env) => needsProbe(env, async (turn, probe) => { + const window = `${turn.ms}ms`; + if (probe.failedRequests <= 1) return pass(`no retry: ${probe.failedRequests} provider request(s) in ${window}`); + const meta = probe.metaKeys.length ? `; _meta keys ${probe.metaKeys.join(', ')}` : ''; + const answered = probe.answerMetaKeys.length ? `; the prompt answer carries _meta ${probe.answerMetaKeys.join(', ')}` : ''; + const outside = Object.keys(probe.otherKinds).length ? `; outside the retries: ${listKinds(probe.otherKinds)}` : ''; + if (!probe.updates) { + return fail(`${probe.failedRequests} provider requests in ${window} and no session/update between the first and the last${outside}${answered}`); + } + return pass(`${probe.updates} session/update(s) between the first and the last of ${probe.failedRequests} provider requests in ${window}: ${listKinds(probe.kinds)}${meta}${outside}${answered}; first ${probe.sample}`); + }), + }, + { + id: 'retry-status-surfaced', + title: "the adapter's trace carries the agent's retry reports", + timeoutMs: (env) => env.timeouts.retryWindowMs + env.timeouts.graceMs * 3, + run: (target, env) => needsProbe(env, async (turn, probe) => { + if (!probe.updates) return skip('the agent sent no update during its retries, so there is nothing to surface'); + const meta = probe.metaKeys.length ? ` (_meta keys ${probe.metaKeys.join(', ')})` : ''; + if (probe.surfaced.length < probe.updates) { + return fail(`the agent sent ${listKinds(probe.kinds)}${meta}; the trace carries ${probe.surfaced.length ? probe.surfaced.join(', ') : 'no event'} from them`); + } + return pass(`${probe.updates} update(s) reached the trace as ${probe.surfaced.join(', ')}`); + }), + }, + { + id: 'stall', + title: "a stalled provider ends within the adapter's timeout as a named failure", + timeoutMs: (env) => env.timeouts.stallMs + env.timeouts.graceMs * 2, + run: (target, env) => needsBinary(env, async () => { + const limit = env.timeouts.stallMs + env.timeouts.graceMs; + const turn = await stallTurn(env); + if (turn.ok) return fail(`a stalled turn reported success: ${quote(turn.result.text)}`); + if (!isNamedFailure(turn.error)) return fail(describeError(turn.error)); + if (turn.error.kind === 'cancelled') return fail(`ended only when the runner cancelled it (${turn.ms}ms)`); + if (turn.ms > limit) return fail(`RunFailure ${turn.error.kind} after ${turn.ms}ms, over the ${limit}ms bound`); + return pass(`RunFailure ${turn.error.kind} after ${turn.ms}ms (timeout ${env.timeouts.stallMs}ms)`); + }), + }, + { + id: 'key-not-in-argv', + title: 'the provider key never appears in argv', + timeoutMs: (env) => env.timeouts.turnMs + env.timeouts.stallMs + env.timeouts.graceMs * 2, + run: (target, env) => needsBinary(env, async () => { + const turns = (await Promise.all(ALL_TURNS.map((start) => start(env)))).filter(Boolean); + const processes = turns.reduce((n, t) => n + t.argv.processes, 0); + const leaks = turns.reduce((n, t) => n + t.argv.leaks, 0); + if (turns.some((t) => !t.argv.available)) return skip('ps is not available on this host'); + if (leaks) return fail(`the key was in the argv of ${leaks} process(es)`); + if (!processes) return skip(`the adapter spawned no child process across ${turns.length} turns`); + return pass(`${processes} process(es) inspected across ${turns.length} turns`); + }), + }, + { + id: 'key-not-in-output', + title: 'the provider key never appears in the reply or the error text', + timeoutMs: (env) => env.timeouts.turnMs + env.timeouts.stallMs + env.timeouts.graceMs * 2, + run: (target, env) => needsBinary(env, async () => { + const key = env.provider.apiKey; + const turns = (await Promise.all(ALL_TURNS.map((start) => start(env)))).filter(Boolean); + const where = []; + turns.forEach((t, index) => { + const surfaces = t.ok + ? [['reply', t.result.text], ['memory summary', t.result.memorySummary]] + : [['error', t.error?.message], ['error detail', t.error?.detail]]; + surfaces.push(['trace', JSON.stringify(t.events)]); + for (const [name, text] of surfaces) { + if (text && String(text).includes(key)) where.push(`${name} of turn ${index + 1}`); + } + }); + return where.length ? fail(`the key appeared in the ${where.join(', ')}`) : pass(`${turns.length} turns clean`); + }), + }, + { + id: 'trace-order', + title: 'the trace has run events in order, when the adapter emits any', + run: (target, env) => needsBinary(env, async () => { + const turns = [['fresh', await freshTurn(env)], ['refusal', await refusalTurn(env)]]; + const emitted = turns.reduce((n, [, t]) => n + t.events.length, 0); + if (!emitted) return skip('the adapter emits no trace events'); + const problems = []; + let withRunEvents = 0; + for (const [name, t] of turns) { + const unknown = t.events.filter((e) => !e.known).map((e) => e.type); + if (unknown.length) problems.push(`${name}: unknown event types ${unknown.join(', ')}`); + const types = t.events.map((e) => e.type); + if (!types.some((type) => type.startsWith('run.'))) continue; + withRunEvents += 1; + const settled = types.filter((type) => type === 'run.settled').length; + if (types[0] !== 'run.started') problems.push(`${name}: first event is ${types[0]}, not run.started`); + if (settled !== 1) problems.push(`${name}: ${settled} run.settled events`); + if (types[types.length - 1] !== 'run.settled') problems.push(`${name}: run.settled is not the last event`); + const outcome = t.events.find((e) => e.type === 'run.settled')?.data?.outcome; + const expected = t.ok ? 'ok' : 'failed'; + if (settled === 1 && outcome !== expected) problems.push(`${name}: settled ${outcome}, expected ${expected}`); + } + if (problems.length) return fail(problems.join('; ')); + if (!withRunEvents) return skip(`${emitted} event(s), none of them run events`); + return pass(`${emitted} events, run events in order on ${withRunEvents} turn(s)`); + }), + }, +]; + +/** Run one check with its own deadline. Never throws; never outlives the deadline. */ +export const runCheck = async (check, target, env, { timeoutMs } = {}) => { + const budget = timeoutMs ?? (typeof check.timeoutMs === 'function' ? check.timeoutMs(env) : check.timeoutMs) + ?? env.timeouts.checkMs; + const started = Date.now(); + let timer; + const deadline = new Promise((resolve) => { + timer = setTimeout(() => resolve({ timedOut: true }), budget); + }); + try { + const outcome = await Promise.race([ + Promise.resolve().then(() => check.run(target, env)).then((value) => ({ value }), (error) => ({ error })), + deadline, + ]); + if (outcome.timedOut) { + env.cancelPending?.(`check ${check.id} ran out of time`); + return { status: 'fail', detail: `check timed out after ${budget}ms`, ms: Date.now() - started }; + } + if (outcome.error) return { status: 'fail', detail: `check threw: ${describeError(outcome.error)}`, ms: Date.now() - started }; + const { status, detail } = outcome.value || {}; + if (!['pass', 'fail', 'skip'].includes(status)) return { status: 'fail', detail: `check returned status ${quote(status)}`, ms: Date.now() - started }; + return { status, detail: String(detail ?? ''), ms: outcome.value.ms ?? Date.now() - started }; + } finally { + clearTimeout(timer); + } +}; diff --git a/packages/seats/compat/harness.js b/packages/seats/compat/harness.js new file mode 100644 index 0000000000..23d53958be --- /dev/null +++ b/packages/seats/compat/harness.js @@ -0,0 +1,267 @@ +/** + * The per-target harness the checks run against. It owns everything a check + * must not improvise: the fake provider, the run env, one memoized turn per + * prompt, the argv watch and the cleanup of anything an adapter left running. + * + * Checks see only the generic Adapter contract (src/contract.js) plus + * ctx.provider, so any adapter — stub, fake-cli, ACP — plugs in unchanged. + */ +import { execFile, spawn as childSpawn, spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { delimiter, dirname, join } from 'node:path'; +import { StringDecoder } from 'node:string_decoder'; +import { RunFailure, runFailure } from '../src/failures.js'; +import { isEventType } from '../src/trace/events.js'; + +const PS_ARGS = ['-A', '-ww', '-o', 'pid=,ppid=,args=']; +const POLL_MS = 100; +const SELF = new RegExp(`^(\\S*/)?ps ${PS_ARGS.join(' ')}\\s*$`); + +// One `ps` listing, reduced to this process's descendants. Argv is matched +// against the key here and never kept, so a leak cannot travel further. +// `ownPids` are the watch's own ps probes. Matching them by argv alone is not +// enough: a probe that has exited but is not yet reaped lists as +// "[ps] ", and one caught mid-exec lists as "[node]" or with this +// runner's argv. Neither is the adapter's. +export const descendantsFrom = (listing, rootPid, key, ownPids = new Set()) => { + const rows = []; + for (const line of String(listing || '').split('\n')) { + const match = /^\s*(\d+)\s+(\d+)\s(.*)$/.exec(line); + if (!match || SELF.test(match[3]) || ownPids.has(Number(match[1]))) continue; + rows.push({ pid: Number(match[1]), ppid: Number(match[2]), leaks: !!key && match[3].includes(key) }); + } + const tree = new Set([rootPid]); + let grew = true; + while (grew) { + grew = false; + for (const row of rows) { + if (!tree.has(row.pid) && tree.has(row.ppid)) { tree.add(row.pid); grew = true; } + } + } + tree.delete(rootPid); + return rows.filter((row) => tree.has(row.pid)); +}; + +/** + * Watch argv of every process this runner descends to while a turn is open. + * Sampled on the adapter's onSpawn (the child is alive at that instant) and on + * a short poll, which catches grandchildren and long-lived agent processes. + */ +export const watchArgv = (key, { rootPid = process.pid } = {}) => { + const seen = new Set(); + const leaked = new Set(); + // Every probe's pid, kept for the watch's life: a poll's listing can show a + // sync probe mid-exec and is only absorbed after spawnSync has returned it. + const probes = new Set(); + let available = true; + let stopped = false; + const absorb = (listing) => { + for (const row of descendantsFrom(listing, rootPid, key, probes)) { + seen.add(row.pid); + if (row.leaks) leaked.add(row.pid); + } + }; + const snapshot = () => { + const probe = spawnSync('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000 }); + if (Number.isInteger(probe.pid)) probes.add(probe.pid); + if (probe.error || probe.status !== 0) { available = false; return; } + absorb(probe.stdout); + }; + let inFlight = false; + const timer = setInterval(() => { + if (inFlight) return; + inFlight = true; + const child = execFile('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000, maxBuffer: 16 * 1024 * 1024 }, (error, stdout) => { + inFlight = false; + // A poll that lands after stop() would change a report already read. + if (stopped) return; + if (error) available = false; else absorb(stdout); + }); + if (Number.isInteger(child.pid)) probes.add(child.pid); + }, POLL_MS); + timer.unref(); + return { + snapshot, + stop() { clearInterval(timer); snapshot(); stopped = true; }, + get report() { return { available, processes: seen.size, leaks: leaked.size }; }, + }; +}; + +/** Kill whatever an adapter left behind under this runner (best effort). */ +export const killDescendants = (rootPid = process.pid) => { + const probe = spawnSync('ps', PS_ARGS, { encoding: 'utf8', timeout: 5000 }); + if (probe.error || probe.status !== 0) return 0; + const rows = descendantsFrom(probe.stdout, rootPid, ''); + for (const row of rows) { + try { process.kill(row.pid, 'SIGKILL'); } catch { /* already gone */ } + } + return rows.length; +}; + +const WIRE_LIMIT = 2000; + +/** + * A spawn that also records the child's newline-delimited JSON on stdin and + * stdout, so a probe sees what the agent said before any adapter read it. + * Lines that are not JSON are not protocol messages and are not kept. The + * listener is attached in the spawning tick, before the adapter's own, so no + * chunk is lost to either reader. `stamp()` adds fields to each record; the + * harness stamps how many provider requests the turn had made by then. + */ +export const tapSpawn = (wire, spawnImpl = childSpawn, { stamp = () => ({}) } = {}) => (command, args, options) => { + const child = spawnImpl(command, args, options); + const started = Date.now(); + const reader = (dir) => { + const decoder = new StringDecoder('utf8'); + let buffer = ''; + return (chunk) => { + buffer += typeof chunk === 'string' ? chunk : decoder.write(chunk); + let at; + while ((at = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, at).trim(); + buffer = buffer.slice(at + 1); + if (!line || wire.length >= WIRE_LIMIT) continue; + try { wire.push({ dir, ms: Date.now() - started, ...stamp(), msg: JSON.parse(line) }); } catch { /* not a protocol line */ } + } + }; + }; + child.stdout?.on('data', reader('in')); + if (child.stdin) { + const write = child.stdin.write.bind(child.stdin); + const out = reader('out'); + child.stdin.write = (chunk, ...rest) => { out(chunk); return write(chunk, ...rest); }; + } + return child; +}; + +// Only names a CLI needs to start: never the host's own secrets. +const runEnv = ({ home, binaryPath }) => { + const xdg = (name) => { const dir = join(home, name); mkdirSync(dir, { recursive: true }); return dir; }; + const path = [binaryPath ? dirname(binaryPath) : null, process.env.PATH].filter(Boolean).join(delimiter); + return { + PATH: path, + HOME: home, + TMPDIR: xdg('tmp'), + XDG_CONFIG_HOME: xdg('.config'), + XDG_DATA_HOME: xdg('.local/share'), + XDG_STATE_HOME: xdg('.local/state'), + XDG_CACHE_HOME: xdg('.cache'), + LANG: process.env.LANG || 'C.UTF-8', + NO_COLOR: '1', + }; +}; + +/** + * @param {object} opts + * @param {import('../src/contract.js').Adapter} opts.adapter + * @param {{ baseURL: string, apiKey: string, model: string, requests: object[] }} opts.provider + * @param {string} [opts.binaryPath] + * @param {{ turnMs: number, stallMs: number, retryWindowMs?: number }} opts.timeouts + * @param {import('../src/contract.js').Adapter} [opts.probeAdapter] the same adapter with + * the target's `probeOptions` applied, for checks that read the agent itself + */ +export const createHarness = ({ adapter, provider, binaryPath = null, timeouts, probeAdapter = null }) => { + const root = mkdtempSync(join(tmpdir(), 'seats-compat-')); + const home = join(root, 'home'); + const cwd = join(root, 'work'); + mkdirSync(home, { recursive: true }); + mkdirSync(cwd, { recursive: true }); + const env = runEnv({ home, binaryPath }); + const pending = new Set(); + const turns = new Map(); + let detection; + + const detect = () => { + if (!detection) { + detection = Promise.resolve().then(() => adapter.detect({ env, binaryPath })) + .then((found) => ({ found: found || null }), (error) => ({ found: null, error })); + } + return detection; + }; + + const runTurn = async (prompt, { sessionId, timeoutMs = timeouts.turnMs, via = adapter, tap = false } = {}) => { + const events = []; + const wire = []; + const tapped = { spawned: false }; + const firstRequest = provider.requests.length; + // How many provider requests this turn had made at a given instant; an + // event and a wire line stamped with it can be placed among the retries. + const stamp = () => ({ requests: provider.requests.length - firstRequest }); + const controller = new AbortController(); + pending.add(controller); + const watch = watchArgv(provider.apiKey); + const trace = { + emit(type, data = {}) { events.push({ seq: events.length + 1, type, data, known: isEventType(type), ...stamp() }); }, + }; + const ctx = { + provider: { baseURL: provider.baseURL, apiKey: provider.apiKey, model: provider.model }, + env: { ...env }, + cwd, + trace, + signal: controller.signal, + onSpawn: () => watch.snapshot(), + timeoutMs, + agentName: 'compat', + ...(sessionId ? { sessionId } : {}), + // ACP adapters take ctx._spawnImpl; one that ignores it leaves the + // wire empty and spawned stays false, which the probe checks report. + ...(tap ? { _spawnImpl: (...args) => { tapped.spawned = true; return tapSpawn(wire, childSpawn, { stamp })(...args); } } : {}), + }; + const started = Date.now(); + let outcome; + try { + const result = await via.spawn(prompt, ctx); + outcome = { ok: true, result: result || {} }; + } catch (error) { + outcome = { ok: false, error }; + } finally { + watch.stop(); + pending.delete(controller); + } + return { + ...outcome, + ms: Date.now() - started, + events, + argv: watch.report, + requests: provider.requests.slice(firstRequest), + ...(tap ? { wire, tapped: tapped.spawned } : {}), + }; + }; + + /** The same named turn is run once per target and shared by every check that reads it. */ + const turn = (name, prompt, opts) => { + if (!turns.has(name)) turns.set(name, runTurn(prompt, opts)); + return turns.get(name); + }; + + /** + * A named turn on the probe adapter with the agent's stdio recorded, or + * null when the target declares no probe. Memoized like turn(). + */ + const probeTurn = (name, prompt, opts = {}) => { + if (!probeAdapter) return null; + const key = `probe:${name}`; + if (!turns.has(key)) turns.set(key, runTurn(prompt, { ...opts, via: probeAdapter, tap: true })); + return turns.get(key); + }; + + // A check that ran out of time leaves its turn open; cancel it so the next + // check starts clean and the adapter is told to stop its child. + const cancelPending = (why) => { + for (const controller of pending) controller.abort(runFailure('cancelled', why)); + }; + + const dispose = () => { + cancelPending('the compatibility run finished'); + rmSync(root, { recursive: true, force: true }); + }; + + return { + adapter, probeAdapter, provider, binaryPath, timeouts, env, cwd, + detect, turn, probeTurn, cancelPending, dispose, + }; +}; + +export const isNamedFailure = (error) => error instanceof RunFailure + || (error?.name === 'RunFailure' && typeof error?.kind === 'string'); diff --git a/packages/seats/compat/install.js b/packages/seats/compat/install.js new file mode 100644 index 0000000000..ee63a8819d --- /dev/null +++ b/packages/seats/compat/install.js @@ -0,0 +1,118 @@ +#!/usr/bin/env node +/** + * Install each enabled target's pinned binary into a job-local prefix. + * + * node compat/install.js --targets compat/targets.json --prefix + * + * Nothing is installed globally. A release asset is downloaded, checked + * against its pinned sha256 BEFORE it is unpacked, and extracted into + * /-/. An npm spec is installed into that directory with + * --ignore-scripts, so no package's install hook runs. A pip spec is installed + * into /venv from wheels only (--only-binary=:all:), so no package's + * setup.py runs either; the interpreter is python3, or COMPAT_PYTHON. A target whose adapter + * module is absent is not installed: it would only be skipped by the runner. + */ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { installDirOf, parseAdapterSpec, platformKey, resolveBinary } from './targets.js'; +import { loadTargets } from './run.js'; + +const sha256 = (buffer) => createHash('sha256').update(buffer).digest('hex'); + +const run = (command, args, opts = {}) => { + const result = spawnSync(command, args, { stdio: ['ignore', 'inherit', 'inherit'], ...opts }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`${command} ${args[0] ?? ''} exited ${result.status}`); +}; + +export const installRelease = async ({ url, sha256: expected }, dir, { fetchImpl = fetch } = {}) => { + if (!/^[0-9a-f]{64}$/.test(String(expected || ''))) throw new Error(`release asset ${url} has no pinned sha256`); + const response = await fetchImpl(url, { redirect: 'follow' }); + if (!response.ok) throw new Error(`download ${url} answered ${response.status}`); + const body = Buffer.from(await response.arrayBuffer()); + const actual = sha256(body); + if (actual !== expected) throw new Error(`sha256 mismatch for ${url}: expected ${expected}, got ${actual}`); + mkdirSync(resolve(dir, '..'), { recursive: true }); + const scratch = mkdtempSync(join(resolve(dir, '..'), '.download-')); + try { + const archive = join(scratch, 'asset'); + writeFileSync(archive, body); + mkdirSync(dir, { recursive: true }); + const flag = /\.tar\.bz2$|\.tbz2?$/.test(url) ? '-xjf' : '-xzf'; + run('tar', [flag, archive, '-C', dir]); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}; + +export const installNpm = (spec, dir) => { + mkdirSync(dir, { recursive: true }); + run('npm', ['install', '--prefix', dir, '--ignore-scripts', '--no-save', '--no-audit', '--no-fund', spec]); +}; + +/** A pinned `name==version` spec, installed from wheels into /venv. */ +export const installPip = (spec, dir, { python = process.env.COMPAT_PYTHON || 'python3', runImpl = run } = {}) => { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*==[A-Za-z0-9][A-Za-z0-9.+_-]*$/.test(String(spec || ''))) { + throw new Error(`pip spec ${JSON.stringify(spec)} must pin one exact version (name==version)`); + } + mkdirSync(dir, { recursive: true }); + const venv = join(dir, 'venv'); + runImpl(python, ['-m', 'venv', venv]); + runImpl(join(venv, 'bin', 'pip'), [ + 'install', '--no-input', '--disable-pip-version-check', '--no-cache-dir', '--only-binary=:all:', spec, + ]); +}; + +export const installTarget = async (target, { prefix, platform = platformKey(), fetchImpl, pipOptions } = {}) => { + const binary = target.binary; + if (target.enabled === false) return { skipped: 'disabled' }; + if (!binary || binary.path) return { skipped: 'no binary to install' }; + if (!existsSync(parseAdapterSpec(target.adapter).path)) return { skipped: `adapter module ${target.adapter} is not present` }; + const resolved = resolveBinary(target, { prefix, platform }); + if (resolved?.missing) return { skipped: resolved.missing }; + const dir = installDirOf(target, prefix); + if (binary.release) await installRelease(binary.release[platform], dir, { fetchImpl }); + else if (binary.npm) installNpm(binary.npm, dir); + else if (binary.pip) installPip(binary.pip, dir, pipOptions); + else return { skipped: 'binary names neither release, npm nor pip' }; + if (!existsSync(resolved.path)) throw new Error(`${target.cli} installed but ${resolved.label} is missing`); + chmodSync(resolved.path, 0o755); + return { installed: resolved.path }; +}; + +const argValue = (argv, name) => { + const at = argv.indexOf(name); + return at === -1 ? null : argv[at + 1] ?? null; +}; + +export const main = async (argv = process.argv.slice(2)) => { + const targetsPath = argValue(argv, '--targets'); + const prefix = argValue(argv, '--prefix'); + if (!targetsPath || !prefix) { + process.stderr.write('usage: node compat/install.js --targets --prefix \n'); + return 2; + } + let failed = 0; + for (const target of loadTargets(resolve(targetsPath))) { + try { + const outcome = await installTarget(target, { prefix: resolve(prefix) }); + process.stdout.write(`${target.cli}@${target.version}: ${outcome.installed ? `installed ${outcome.installed}` : `skipped (${outcome.skipped})`}\n`); + } catch (error) { + // One CLI failing to install must not hide the rest of the matrix; the + // runner reports it as "binary not installed". + failed += 1; + process.stderr.write(`${target.cli}@${target.version}: install failed: ${error?.message || error}\n`); + } + } + return failed ? 1 : 0; +}; + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().then((code) => process.exit(code), (error) => { + process.stderr.write(`compat install failed: ${error?.stack || error}\n`); + process.exit(1); + }); +} diff --git a/packages/seats/compat/render.js b/packages/seats/compat/render.js new file mode 100644 index 0000000000..dd7f74445d --- /dev/null +++ b/packages/seats/compat/render.js @@ -0,0 +1,171 @@ +/** + * Render a results document as one self-contained HTML page: no external + * scripts, styles, fonts or images, so the artifact opens anywhere and a + * Pages deploy needs nothing but this file. Every value from the results is + * escaped — detail strings carry CLI and provider output. + */ +const ESCAPES = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; +export const escapeHtml = (value) => String(value ?? '').replace(/[&<>"']/g, (c) => ESCAPES[c]); + +const MARK = { pass: '✓', fail: '✗', skip: '–', limit: '!' }; +const STATUSES = new Set(['pass', 'fail', 'skip']); + +/** A failing row is a limit when its target declares one for that check (run.js limitOf). */ +export const isLimitRow = (row) => row?.status === 'fail' && typeof row.limit?.note === 'string' && !!row.limit.note; +const statusOf = (row) => { + if (isLimitRow(row)) return 'limit'; + return row && STATUSES.has(row.status) ? row.status : 'missing'; +}; + +/** "limit of opencode@1.19.4: . Fixed in ." as plain text. */ +export const limitText = (row) => { + const { note, fixedIn } = row?.limit || {}; + if (!note) return ''; + const where = row.target ? ` of ${row.target}` : ' of this version'; + return `limit${where}: ${note}${fixedIn ? `. Fixed in ${fixedIn}` : ''}`; +}; + +const STYLE = ` +:root{--bg:#fbfbfa;--fg:#1c1c1a;--muted:#6b6b66;--line:#e3e2de;--card:#fff; +--pass:#1f7a3d;--pass-bg:#e6f4ea;--fail:#b3261e;--fail-bg:#fbe9e7;--skip:#6b6b66;--skip-bg:#efeeea; +--limit:#8a5300;--limit-bg:#fdf0d5;} +@media (prefers-color-scheme:dark){:root{--bg:#161615;--fg:#ecebe8;--muted:#a3a29c;--line:#34332f;--card:#1f1e1c; +--pass:#7fd79a;--pass-bg:#173523;--fail:#ff9a8f;--fail-bg:#3d1c19;--skip:#a3a29c;--skip-bg:#2a2926; +--limit:#f2c46b;--limit-bg:#3a2c10;}} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",system-ui,sans-serif} +main{max-width:1120px;margin:0 auto;padding:24px 16px 48px} +h1{font-size:22px;margin:0 0 4px} +.meta{color:var(--muted);font-size:13px;margin:0 0 16px;overflow-wrap:anywhere} +.totals{display:flex;gap:8px;flex-wrap:wrap;margin:0 0 16px;padding:0;list-style:none} +.totals li{padding:2px 10px;border-radius:999px;font-size:13px;font-weight:600} +.scroll{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--card)} +table{border-collapse:collapse;width:100%;font-size:14px} +th,td{padding:8px 10px;border-bottom:1px solid var(--line);text-align:center;vertical-align:top} +thead th{font-size:12px;color:var(--muted);font-weight:600;white-space:nowrap} +th.t,td.t{text-align:left;position:sticky;left:0;background:var(--card);white-space:nowrap;z-index:1} +td.t small{display:block;color:var(--muted);font-weight:400;white-space:normal;max-width:220px} +.pass{color:var(--pass);background:var(--pass-bg)}.fail{color:var(--fail);background:var(--fail-bg)} +.skip,.missing{color:var(--skip);background:var(--skip-bg)}.limit{color:var(--limit);background:var(--limit-bg)} +td.c{font-weight:700;min-width:36px;position:relative} +tr.d td{text-align:left;background:var(--bg);font-size:13px} +tr.d ul{margin:0;padding-left:18px} +/* Details stay readable without scrolling the grid sideways on a phone. */ +.dd{position:sticky;left:10px;max-width:min(100%,calc(100vw - 54px))}tr.d li{overflow-wrap:anywhere} +tr.d code,details code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px} +details{margin-top:4px}summary{cursor:pointer;color:var(--muted)} +/* .sr stays inside its positioned cell; unanchored, it widened the page at 390px. */ +.sr{position:absolute;left:0;top:0;width:1px;height:1px;overflow:hidden;clip-path:inset(50%);white-space:nowrap} +ol.legend{padding-left:22px;color:var(--muted);font-size:13px} +.up{border:1px solid var(--line);border-radius:10px;background:var(--card);padding:12px 14px;margin:0 0 10px;overflow-wrap:anywhere} +.up h3{font-size:15px;margin:0 0 6px}.up p,.up ul{margin:4px 0;font-size:14px}.up ul{padding-left:18px} +.badge{display:inline-block;padding:1px 8px;border-radius:999px;font-size:12px;font-weight:600;margin-right:6px} +.worked,.fixed{color:var(--pass);background:var(--pass-bg)}.open{color:var(--fail);background:var(--fail-bg)} +.up code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:12px} +@media (max-width:480px){main{padding:16px}th,td{padding:6px}td.t small{max-width:140px}} +`; + +const cell = (row, check) => { + const status = statusOf(row); + const limit = status === 'limit' ? ` (${limitText(row)})` : ''; + const label = `${check.title}: ${status}${limit}${row?.detail ? ` — ${row.detail}` : ''}`; + return `${escapeHtml(status)}`; +}; + +// A limit leads with what the version cannot do and where it is fixed; a +// declared limit on a passing row says it did not reproduce this run. +const limitNote = (row) => { + if (!row?.limit?.note) return ''; + if (isLimitRow(row)) return ` ${escapeHtml(limitText(row))}`; + if (row.status === 'pass') return ` (declared ${escapeHtml(limitText(row))}; it did not reproduce)`; + return ''; +}; + +const detailItem = (check, row) => `
  • ${escapeHtml(check.id)} ${escapeHtml(statusOf(row))}${limitNote(row)}${row?.detail ? `: ${escapeHtml(row.detail)}` : ''}${Number.isFinite(row?.ms) && row.ms ? ` (${escapeHtml(row.ms)}ms)` : ''}
  • `; + +const targetRows = (target, checks, rows) => { + const byCheck = new Map(rows.filter((r) => r.target === target.target).map((r) => [r.check, r])); + const failing = checks.filter((c) => ['fail', 'limit'].includes(statusOf(byCheck.get(c.id))) || !byCheck.has(c.id)); + const note = target.note || (target.requires ? `requires ${target.requires}` : ''); + const head = `${escapeHtml(target.target)}${note ? `${escapeHtml(note)}` : ''}${checks.map((c) => cell(byCheck.get(c.id), c)).join('')}`; + const all = `
    All ${checks.length} results
      ${checks.map((c) => detailItem(c, byCheck.get(c.id))).join('')}
    `; + const failures = failing.length ? `
      ${failing.map((c) => detailItem(c, byCheck.get(c.id))).join('')}
    ` : ''; + return `${head}
    ${failures}${all}
    `; +}; + +// Only an https link becomes an anchor; anything else stays text. +const reference = (url) => (/^https:\/\/[^\s"<>]+$/.test(url) + ? `${escapeHtml(url)}` : escapeHtml(url)); + +/** + * One upstream bug: its status, the workaround (or the planned one) and where + * it lives, the tests that prove it, what this run's matrix saw, and the + * upstream links as references only. + */ +const BADGE = { 'worked around': 'worked', 'fixed upstream': 'fixed' }; + +export const upstreamItem = (row) => { + const worked = row?.status === 'worked around'; + const fixed = row?.status === 'fixed upstream'; + const w = row?.workaround || {}; + const where = [w.ref && `${escapeHtml(w.ref)}`, w.where && escapeHtml(w.where)].filter(Boolean).join(' · '); + const tests = Array.isArray(w.tests) && w.tests.length + ? `
      ${w.tests.map((t) => `
    • ${escapeHtml(t)}
    • `).join('')}
    ` + : (fixed ? '' : '

    No test proves it yet.

    '); + const fixedLine = fixed ? `

    Fixed upstream in ${escapeHtml(row.fixed?.version)}: ${escapeHtml(row.fixed?.verified)}

    \n` : ''; + const label = worked ? 'Workaround' : fixed ? 'Our workaround, for older versions' : 'Planned workaround'; + const matrix = row?.matrix ? `

    This run: ${escapeHtml(row.matrix.target)} · ${escapeHtml(row.matrix.check)} ${escapeHtml(row.matrix.observed || 'not run')}

    ` : ''; + const refs = Array.isArray(row?.references) && row.references.length + ? `

    Upstream (reference, not the fix): ${row.references.map(reference).join(' · ')}

    ` : ''; + return `

    ${BADGE[row?.status] ? escapeHtml(row.status) : 'open'}${escapeHtml(row?.cli)}${row?.versions ? ` ${escapeHtml(row.versions)}` : ''}

    +

    ${escapeHtml(row?.bug)}

    +${row?.downgraded ? `

    ${escapeHtml(row.downgraded)}

    ` : ''}${fixedLine}

    ${label}: ${escapeHtml(w.name)}${where ? ` (${where})` : ''}

    +${tests}${matrix}${refs}
    `; +}; + +const upstreamSection = (upstream) => { + if (!Array.isArray(upstream) || !upstream.length) return ''; + const worked = upstream.filter((r) => r.status === 'worked around').length; + const fixed = upstream.filter((r) => r.status === 'fixed upstream').length; + return `

    Upstream bugs

    +

    ${worked} worked around · ${fixed} fixed upstream · ${upstream.length - worked - fixed} open. A row is worked around only when its workaround is on a pushed commit and a test there proves it, and fixed upstream only when we checked the fixing release ourselves.

    +${upstream.map(upstreamItem).join('\n')}`; +}; + +/** @param {object} results the document run.js writes as results.json */ +export const renderMatrix = (results) => { + const checks = Array.isArray(results?.checks) ? results.checks : []; + const rows = Array.isArray(results?.rows) ? results.rows : []; + const targets = Array.isArray(results?.targets) ? results.targets : []; + const host = results?.host || {}; + const count = (s) => rows.filter((r) => statusOf(r) === s).length; + return ` + + + + +Seats compatibility + + + +
    +

    Seats compatibility

    +

    Generated ${escapeHtml(results?.generatedAt)} on ${escapeHtml(host.os)} ${escapeHtml(host.arch)}, node ${escapeHtml(host.node)} · schema ${escapeHtml(results?.schema)}

    +
    • ${count('pass')} pass
    • ${count('fail')} fail
    • ${count('limit')} limit
    +
    + +${checks.map((c, i) => ``).join('')} + +${targets.map((t) => targetRows(t, checks, rows)).join('\n')} + +
    CLI${i + 1}
    +
    +${upstreamSection(results?.upstream)} +

    Checks

    +

    ${MARK.pass} pass · ${MARK.fail} fail · ${MARK.limit} limit of that CLI version (a failure its target declares, with where it is fixed) · ${MARK.skip} skip

    +
      ${checks.map((c) => `
    1. ${escapeHtml(c.id)} ${escapeHtml(c.title)}
    2. `).join('')}
    +
    + + +`; +}; diff --git a/packages/seats/compat/run.js b/packages/seats/compat/run.js new file mode 100644 index 0000000000..e270807043 --- /dev/null +++ b/packages/seats/compat/run.js @@ -0,0 +1,199 @@ +#!/usr/bin/env node +/** + * The compatibility matrix runner. + * + * node compat/run.js --targets compat/targets.json --out [--prefix ] + * + * Starts the fake provider, runs every enabled target against every check, + * and writes /results.json and /index.html. A failing check is data, + * not a crash: the exit code is 0 whenever the matrix was produced. + * `--prefix` (or COMPAT_PREFIX) is where compat/install.js put the binaries. + */ +import { randomBytes } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { arch, platform, release } from 'node:os'; +import { delimiter, dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { startFakeProvider } from '../test-support/fake-provider.js'; +import { CHECKS, runCheck } from './checks.js'; +import { createHarness, killDescendants } from './harness.js'; +import { isLimitRow, renderMatrix } from './render.js'; +import { loadAdapter, resolveBinary, targetLabel } from './targets.js'; +import { loadUpstream, upstreamForResults } from './upstream.js'; + +export { isLimitRow }; +export const RESULTS_SCHEMA = 'seats-compat/1'; +export const DEFAULT_TIMEOUTS = Object.freeze({ + turnMs: 30000, stallMs: 8000, graceMs: 5000, checkMs: 45000, retryWindowMs: 20000, +}); + +/** + * A target may declare `limits: { : { note, fixedIn? } }`: a check + * that fails for a reason that belongs to that CLI version, not to the + * adapter. The row keeps its real status; the limit rides along so the page + * can say "limit of this version" and where it is fixed, and so a declared + * limit that stops reproducing is visible as a pass that carries one. + */ +export const limitOf = (target, checkId) => { + const limit = target?.limits?.[checkId]; + if (!limit || typeof limit.note !== 'string' || !limit.note) return null; + return { note: limit.note, ...(typeof limit.fixedIn === 'string' && limit.fixedIn ? { fixedIn: limit.fixedIn } : {}) }; +}; + +/** pass / fail / skip / limit counts; `fail` counts only failures no limit explains. */ +export const totalsOf = (rows) => ({ + pass: rows.filter((r) => r.status === 'pass').length, + fail: rows.filter((r) => r.status === 'fail' && !isLimitRow(r)).length, + skip: rows.filter((r) => r.status === 'skip').length, + limit: rows.filter(isLimitRow).length, +}); + +const rowsFor = (target, statusOf) => CHECKS.map((check) => { + const { status, detail } = statusOf(check); + return { target: targetLabel(target), cli: target.cli, version: target.version, check: check.id, status, detail, ms: 0 }; +}); + +const timeoutsFor = (target, overrides) => { + const merged = { ...DEFAULT_TIMEOUTS, ...(target.timeouts || {}), ...(overrides || {}) }; + if (!overrides?.checkMs && !target.timeouts?.checkMs) merged.checkMs = merged.turnMs + 15000; + return merged; +}; + +// Adapters whose detect() reads the host PATH see the pinned binary first, +// for the length of this target only. +const withPath = async (binaryPath, fn) => { + if (!binaryPath) return fn(); + const before = process.env.PATH; + process.env.PATH = [dirname(binaryPath), before].filter(Boolean).join(delimiter); + try { return await fn(); } finally { process.env.PATH = before; } +}; + +/** Run one target through the suite. Always returns one row per check. */ +export const runTarget = async (target, { provider, prefix, checks = CHECKS, timeouts, log = () => {} } = {}) => { + const label = targetLabel(target); + const all = (status, detail) => rowsFor(target, () => ({ status, detail })).filter((row) => checks.some((c) => c.id === row.check)); + if (target.enabled === false) return all('skip', 'target disabled'); + const binary = resolveBinary(target, { prefix }); + if (binary?.missing) return all('skip', binary.missing); + const loaded = await loadAdapter(target, { binaryPath: binary?.path }); + if (loaded.skip) return all('skip', loaded.skip); + if (loaded.error) return all('fail', loaded.error); + if (binary?.path && !existsSync(binary.path)) { + return checks.map((check) => ({ + target: label, cli: target.cli, version: target.version, check: check.id, ms: 0, + ...(check.id === 'detect' ? { status: 'fail', detail: `binary not installed at ${binary.label}` } : { status: 'skip', detail: 'binary not installed' }), + })); + } + // The probe instance is the same export built with probeOptions on top, so + // a check can see the agent with nothing of the adapter's own in between. + let probeAdapter = null; + if (target.probeOptions) { + const probe = await loadAdapter({ ...target, options: { ...(target.options || {}), ...target.probeOptions } }, { binaryPath: binary?.path }); + if (probe.adapter && probe.adapter !== loaded.adapter) probeAdapter = probe.adapter; + else log(`${label}: probeOptions ignored (${probe.error || probe.skip || 'the export is an adapter, not a factory'})`); + } + const harness = createHarness({ + adapter: loaded.adapter, provider, binaryPath: binary?.path, timeouts: timeoutsFor(target, timeouts), probeAdapter, + }); + const rows = []; + try { + await withPath(binary?.path, async () => { + for (const check of checks) { + const outcome = await runCheck(check, { ...target, binaryLabel: binary?.label }, harness); + log(`${label} ${check.id}: ${outcome.status}${outcome.detail ? ` — ${outcome.detail}` : ''}`); + const limit = limitOf(target, check.id); + rows.push({ target: label, cli: target.cli, version: target.version, check: check.id, ...outcome, ...(limit ? { limit } : {}) }); + } + }); + } finally { + harness.dispose(); + killDescendants(); + } + return rows; +}; + +export const hostInfo = () => ({ os: `${platform()} ${release()}`, platform: platform(), arch: arch(), node: process.version }); + +/** Run every target and return the results document (not yet written). */ +export const runMatrix = async (targets, { prefix, checks = CHECKS, timeouts, log, upstream = [], now = () => new Date() } = {}) => { + const fake = await startFakeProvider(); + // A key-shaped value that is fake by construction: the leak checks look for + // exactly this string, and no real provider would accept it. + const provider = { ...fake, apiKey: `sk-compat-${randomBytes(16).toString('hex')}` }; + const rows = []; + try { + for (const target of targets) rows.push(...await runTarget(target, { provider, prefix, checks, timeouts, log })); + } finally { + await fake.close(); + } + return { + schema: RESULTS_SCHEMA, + generatedAt: now().toISOString(), + host: hostInfo(), + checks: checks.map(({ id, title }) => ({ id, title })), + targets: targets.map((t) => ({ + target: targetLabel(t), cli: t.cli, version: t.version, adapter: t.adapter, + enabled: t.enabled !== false, requires: t.requires || null, note: t.note || null, + })), + rows, + totals: totalsOf(rows), + upstream: upstreamForResults(upstream, rows), + }; +}; + +export const writeOutputs = (results, outDir) => { + mkdirSync(outDir, { recursive: true }); + const json = join(outDir, 'results.json'); + const html = join(outDir, 'index.html'); + writeFileSync(json, `${JSON.stringify(results, null, 2)}\n`); + writeFileSync(html, renderMatrix(results)); + return { json, html }; +}; + +export const loadTargets = (path) => { + const parsed = JSON.parse(readFileSync(path, 'utf8')); + const targets = Array.isArray(parsed) ? parsed : parsed?.targets; + if (!Array.isArray(targets)) throw new Error(`${path} must hold an array of targets`); + for (const t of targets) { + if (!t?.cli || !t?.version || !t?.adapter) throw new Error(`every target needs cli, version and adapter: ${JSON.stringify(t)}`); + } + return targets; +}; + +const argValue = (argv, name) => { + const at = argv.indexOf(name); + return at === -1 ? null : argv[at + 1] ?? null; +}; + +export const main = async (argv = process.argv.slice(2)) => { + const targetsPath = argValue(argv, '--targets'); + const out = argValue(argv, '--out'); + if (!targetsPath || !out) { + process.stderr.write('usage: node compat/run.js --targets --out [--prefix ] [--upstream ]\n'); + return 2; + } + const prefix = argValue(argv, '--prefix') || process.env.COMPAT_PREFIX || null; + // Upstream bug rows live beside the targets unless --upstream names a file. + const named = argValue(argv, '--upstream'); + const upstreamPath = resolve(named || join(dirname(resolve(targetsPath)), 'upstream.json')); + if (named && !existsSync(upstreamPath)) { + process.stderr.write(`no upstream file at ${upstreamPath}\n`); + return 2; + } + const results = await runMatrix(loadTargets(resolve(targetsPath)), { + prefix, upstream: loadUpstream(upstreamPath), log: (line) => process.stdout.write(`${line}\n`), + }); + const paths = writeOutputs(results, resolve(out)); + const { pass, fail, skip, limit } = results.totals; + process.stdout.write(`pass ${pass} fail ${fail} limit ${limit} skip ${skip}\n${paths.json}\n${paths.html}\n`); + return 0; +}; + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + // Exit explicitly: an adapter that ignored cancellation must not keep the + // nightly job alive after the matrix is written. + main().then((code) => process.exit(code), (error) => { + process.stderr.write(`compat run failed: ${error?.stack || error}\n`); + process.exit(1); + }); +} diff --git a/packages/seats/compat/targets.js b/packages/seats/compat/targets.js new file mode 100644 index 0000000000..b8cf102a0c --- /dev/null +++ b/packages/seats/compat/targets.js @@ -0,0 +1,77 @@ +/** + * Targets are data: `{ cli, version, adapter: 'module#export', binary?, enabled, + * requires?, options?, timeouts?, note? }`. This module turns one entry into + * an adapter and a binary path, or into the reason it cannot run. + * + * Module specs resolve against the package root, so `src/adapters/stub.js` + * means packages/seats/src/adapters/stub.js wherever the runner is started. + * The export may be an adapter object or a factory taking `options`. + */ +import { existsSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const PACKAGE_ROOT = resolve(fileURLToPath(new URL('..', import.meta.url))); +export const platformKey = (platform = process.platform, arch = process.arch) => `${platform}-${arch}`; + +export const parseAdapterSpec = (spec) => { + const [file, name = 'default'] = String(spec || '').split('#'); + if (!file) throw new Error(`adapter spec ${JSON.stringify(spec)} names no module`); + return { file, exportName: name, path: isAbsolute(file) ? file : join(PACKAGE_ROOT, file) }; +}; + +/** Where install.js puts a target, and where run.js looks for it. */ +export const installDirOf = (target, prefix) => join(prefix, `${target.cli}-${target.version}`); + +const expand = (value) => String(value) + .replace(/^~(?=$|\/)/, homedir()) + .replace(/\$\{(\w+)\}/g, (_, name) => process.env[name] ?? ''); + +/** + * The binary a target runs, as `{ path, label }` (label is prefix-relative so a + * published page never carries a host's home directory), `null` when the + * target names none, or `{ missing }` when it cannot run on this host. + */ +export const resolveBinary = (target, { prefix, platform = platformKey() } = {}) => { + const binary = target.binary; + if (!binary) return null; + if (binary.path) return { path: expand(binary.path), label: binary.path }; + const bin = typeof binary.bin === 'string' ? binary.bin : binary.bin?.[platform]; + if (!bin) return { missing: `no ${target.cli} binary is pinned for ${platform}` }; + if (binary.release && !binary.release[platform]) return { missing: `no ${target.cli} release asset is pinned for ${platform}` }; + if (!prefix) return { missing: 'no install prefix given (--prefix or COMPAT_PREFIX)' }; + return { path: join(installDirOf(target, expand(prefix)), bin), label: `/${target.cli}-${target.version}/${bin}` }; +}; + +/** + * @returns {Promise<{ adapter } | { skip: string } | { error: string }>} + */ +export const loadAdapter = async (target, { binaryPath } = {}) => { + let spec; + try { spec = parseAdapterSpec(target.adapter); } catch (error) { return { error: error.message }; } + if (!existsSync(spec.path)) { + const why = target.requires ? ` (requires: ${target.requires})` : ''; + return { skip: `adapter module ${spec.file} is not present${why}` }; + } + let mod; + try { mod = await import(pathToFileURL(spec.path).href); } catch (error) { + return { error: `importing ${spec.file} failed: ${error?.message || error}` }; + } + const exported = mod[spec.exportName]; + if (exported === undefined) return { error: `${spec.file} has no export ${spec.exportName}` }; + let adapter = exported; + if (typeof exported === 'function') { + try { + adapter = await exported({ ...(target.options || {}), ...(binaryPath ? { binaryPath } : {}) }); + } catch (error) { + return { error: `${spec.file}#${spec.exportName}(options) threw: ${error?.message || error}` }; + } + } + if (!adapter || typeof adapter.spawn !== 'function' || typeof adapter.detect !== 'function') { + return { error: `${target.adapter} is not an adapter (needs detect() and spawn())` }; + } + return { adapter }; +}; + +export const targetLabel = (target) => `${target.cli}@${target.version}`; diff --git a/packages/seats/compat/targets.json b/packages/seats/compat/targets.json new file mode 100644 index 0000000000..b34dd846e7 --- /dev/null +++ b/packages/seats/compat/targets.json @@ -0,0 +1,260 @@ +[ + { + "cli": "stub", + "version": "0.0.0", + "adapter": "src/adapters/stub.js#default", + "enabled": true, + "note": "reference no-op adapter: it never calls a provider, so the provider checks fail by design", + "timeouts": { + "turnMs": 5000, + "stallMs": 2000 + } + }, + { + "cli": "fake-cli", + "version": "1.0.0", + "adapter": "test-support/fake-cli-adapter.js#default", + "enabled": true, + "note": "a real child process speaking to the fake provider; the runner's own control", + "timeouts": { + "turnMs": 10000, + "stallMs": 2000 + } + }, + { + "cli": "goose", + "version": "1.56.0", + "adapter": "src/adapters/acp-launch.js#createAcpGooseAdapter", + "requires": "acp", + "enabled": true, + "probeOptions": { + "watchProvider": false + }, + "limits": { + "retry-status": { + "note": "retries a provider 5xx with backoff and sends no session/update while it does; its error text arrives only after it gives up, as an agent_message_chunk followed by end_turn" + } + }, + "binary": { + "bin": "goose", + "release": { + "linux-x64": { + "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "225a929b35b5130fb691eafdb50be236f2c0a150019b53973775938f8f19c281" + }, + "linux-arm64": { + "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-aarch64-unknown-linux-gnu.tar.gz", + "sha256": "26bcb7d3f55aac485a91d81cbdecd8d90e5af1acadeca6e580b4d58b64a888e4" + }, + "darwin-arm64": { + "url": "https://github.com/aaif-goose/goose/releases/download/v1.56.0/goose-aarch64-apple-darwin.tar.gz", + "sha256": "52f3f8d62e3d144c5de70c5c471de39d8b92c24d41d5b3436eb8bab11ba4bc24" + } + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "opencode", + "version": "1.19.4", + "adapter": "src/adapters/acp-launch.js#createAcpOpencodeAdapter", + "requires": "acp", + "enabled": true, + "note": "opencode-ai, the 1.x line", + "probeOptions": { + "watchProvider": false + }, + "limits": { + "retry-status": { + "note": "retries a provider 5xx with backoff (about 73 s before it gives up) and sends no session/update while it does, then answers session/prompt with -32603 and no HTTP status", + "fixedIn": "@opencode/cli ≥2.0.15" + } + }, + "binary": { + "npm": "opencode-ai@1.19.4", + "bin": { + "linux-x64": "node_modules/opencode-linux-x64/bin/opencode", + "linux-arm64": "node_modules/opencode-linux-arm64/bin/opencode", + "darwin-arm64": "node_modules/opencode-darwin-arm64/bin/opencode" + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "opencode-v2", + "version": "2.1.4", + "adapter": "src/adapters/acp-launch.js#createAcpOpencodeAdapter", + "requires": "acp", + "enabled": true, + "note": "@opencode/cli, the 2.x line: a separate npm package whose launcher needs its postinstall, so the platform binary is run directly", + "probeOptions": { + "watchProvider": false + }, + "binary": { + "npm": "@opencode/cli@2.1.4", + "bin": { + "linux-x64": "node_modules/@opencode/cli-linux-x64/bin/opencode", + "linux-arm64": "node_modules/@opencode/cli-linux-arm64/bin/opencode", + "darwin-arm64": "node_modules/@opencode/cli-darwin-arm64/bin/opencode" + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "copilot", + "version": "1.0.103", + "adapter": "src/adapters/copilot.js#createCopilotAdapter", + "enabled": true, + "note": "GitHub Copilot CLI, @github/copilot: BYOK through COPILOT_PROVIDER_BASE_URL (type openai) with COPILOT_OFFLINE, so no GitHub sign-in", + "binary": { + "npm": "@github/copilot@1.0.103", + "bin": { + "linux-x64": "node_modules/@github/copilot-linux-x64/copilot", + "linux-arm64": "node_modules/@github/copilot-linux-arm64/copilot", + "darwin-arm64": "node_modules/@github/copilot-darwin-arm64/copilot" + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "gemini", + "version": "0.65.0", + "adapter": "src/adapters/provider-unsupported.js#createGeminiAdapter", + "enabled": true, + "note": "Google Gemini CLI, @google/gemini-cli: speaks only the Gemini API, so every turn check reports provider_unsupported", + "binary": { + "npm": "@google/gemini-cli@0.65.0", + "bin": "node_modules/.bin/gemini" + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "qwen-code", + "version": "0.26.1", + "adapter": "src/adapters/qwen.js#createQwenAdapter", + "enabled": true, + "note": "Qwen Code, @qwen-code/qwen-code: OpenAI auth through OPENAI_BASE_URL; the production qwen adapter runs the same invocation", + "limits": { + "provider-5xx": { + "note": "retries a provider 503 and has not given up when the 60 s turn deadline ends it (22 requests in 60 s, 0.26.1)" + } + }, + "binary": { + "npm": "@qwen-code/qwen-code@0.26.1", + "bin": "node_modules/.bin/qwen" + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "cline", + "version": "3.0.73", + "adapter": "src/adapters/cline.js#createClineAdapter", + "enabled": true, + "note": "Cline CLI, the cline npm package: an openai-compatible provider written to its settings store", + "limits": { + "provider-5xx": { + "note": "retries a provider 503 and has not given up when the 60 s turn deadline ends it (3.0.73)" + }, + "resume-session": { + "note": "--id with --json refuses every prompt form (positional, after --, piped stdin) with \"JSON output mode requires a prompt argument or piped stdin\" (3.0.73)" + } + }, + "binary": { + "npm": "cline@3.0.73", + "bin": { + "linux-x64": "node_modules/@cline/cli-linux-x64/bin/cline", + "linux-arm64": "node_modules/@cline/cli-linux-arm64/bin/cline", + "darwin-arm64": "node_modules/@cline/cli-darwin-arm64/bin/cline" + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "aider", + "version": "0.86.2", + "adapter": "src/adapters/aider.js#createAiderAdapter", + "enabled": true, + "note": "Aider, aider-chat on PyPI: installed into a venv from wheels only; needs python 3.10 to 3.12 on PATH as python3 (or COMPAT_PYTHON)", + "limits": { + "refusal": { + "note": "litellm reads a 400 content-policy refusal as retryable: aider asks the provider 9 times over about 64 s with backoff, then prints the refusal and stops (0.86.2)" + } + }, + "binary": { + "pip": "aider-chat==0.86.2", + "bin": "venv/bin/aider" + }, + "timeouts": { + "turnMs": 120000, + "stallMs": 15000 + } + }, + { + "cli": "amp", + "version": "0.0.1792958443-g695471", + "adapter": "src/adapters/provider-unsupported.js#createAmpAdapter", + "enabled": true, + "note": "Amp, @ampcode/cli: model calls go through the Amp service, so every turn check reports provider_unsupported", + "binary": { + "npm": "@ampcode/cli@0.0.1792958443-g695471", + "bin": { + "linux-x64": "node_modules/@ampcode/cli-linux-x64/amp", + "linux-arm64": "node_modules/@ampcode/cli-linux-arm64/amp", + "darwin-arm64": "node_modules/@ampcode/cli-darwin-arm64/amp" + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + }, + { + "cli": "cursor", + "version": "2026.10.22-27dcf5f", + "adapter": "src/adapters/provider-unsupported.js#createCursorAdapter", + "enabled": true, + "note": "Cursor CLI, cursor-agent: model calls go through Cursor's backend, so every turn check reports provider_unsupported", + "binary": { + "bin": "dist-package/cursor-agent", + "release": { + "linux-x64": { + "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/linux/x64/agent-cli-package.tar.gz", + "sha256": "85296944e4600a83c4b764f578d1502bf9c416db200d33a593e8f3ecabfdc294" + }, + "linux-arm64": { + "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/linux/arm64/agent-cli-package.tar.gz", + "sha256": "22a36df9720830668ccd4d707e5190204ab9c8a0efb2f068cc914895f804d3ec" + }, + "darwin-arm64": { + "url": "https://downloads.cursor.com/lab/2026.10.22-27dcf5f/darwin/arm64/agent-cli-package.tar.gz", + "sha256": "2f5b0a3dd5fb951cecffac813867d8a08d810e59837d59fd5f12d69b922398d7" + } + } + }, + "timeouts": { + "turnMs": 60000, + "stallMs": 15000 + } + } +] diff --git a/packages/seats/compat/upstream.js b/packages/seats/compat/upstream.js new file mode 100644 index 0000000000..60fb219162 --- /dev/null +++ b/packages/seats/compat/upstream.js @@ -0,0 +1,102 @@ +/** + * Known upstream bugs and what we do about each, as data + * (compat/upstream.json): `{ id, cli, versions, bug, status, workaround: + * { name, ref, where, tests[] }, fixed?: { version, verified }, + * matrix?: { target, check }, references[] }`. + * + * We do not wait on upstream. Every row is "worked around", "fixed upstream" + * or "open", and the rule is enforced here, not trusted from the file: a row + * reads as worked around only when its workaround names the pushed commit it + * lives on (`ref`, a full sha) and at least one test there that proves it. A + * row reads as fixed upstream only when it names the release that fixed it + * (`fixed.version`), says how we verified that release ourselves + * (`fixed.verified`), and links the upstream issue. Anything less is open, + * with the planned workaround named. An upstream issue is a reference, never + * the proof, so a link alone changes nothing. + */ +import { existsSync, readFileSync } from 'node:fs'; +import { isLimitRow } from './render.js'; + +export const UPSTREAM_STATUSES = Object.freeze(['worked around', 'fixed upstream', 'open']); + +const text = (value) => (typeof value === 'string' ? value.trim() : ''); +const SHA = /^[0-9a-f]{40}$/; + +const HTTPS = /^https:\/\/\S+$/; + +/** "worked around" only with a commit and a test; otherwise "open". */ +const workedAround = (row) => { + const workaround = row?.workaround || {}; + return SHA.test(text(workaround.ref)) && Array.isArray(workaround.tests) && workaround.tests.some((t) => text(t)); +}; + +/** "fixed upstream" only with the fixing release, our own check of it, and the issue link. */ +const fixedUpstream = (row) => Boolean(text(row?.fixed?.version) && text(row?.fixed?.verified) + && Array.isArray(row?.references) && row.references.some((r) => HTTPS.test(text(r)))); + +export const upstreamStatusOf = (row) => { + if (row?.status === 'worked around' && workedAround(row)) return 'worked around'; + if (row?.status === 'fixed upstream' && fixedUpstream(row)) return 'fixed upstream'; + return 'open'; +}; + +const DOWNGRADED = { + 'worked around': 'listed as worked around, but names no commit or no test that proves it', + 'fixed upstream': 'listed as fixed upstream, but names no fixing release, no check of it, or no upstream link', +}; + +/** + * Normalise rows for the results document. A row that claims "worked around" + * without the proof is downgraded and says why, so the page never repeats an + * unproven claim. + */ +export const normaliseUpstream = (rows) => (Array.isArray(rows) ? rows : []).map((row) => { + const status = upstreamStatusOf(row); + const workaround = row?.workaround || {}; + return { + id: text(row?.id), + cli: text(row?.cli), + versions: text(row?.versions), + bug: text(row?.bug), + status, + ...(DOWNGRADED[row?.status] && status === 'open' ? { downgraded: DOWNGRADED[row.status] } : {}), + workaround: { + name: text(workaround.name), + ref: text(workaround.ref), + where: text(workaround.where), + tests: (Array.isArray(workaround.tests) ? workaround.tests : []).map(text).filter(Boolean), + }, + ...(status === 'fixed upstream' ? { fixed: { version: text(row.fixed.version), verified: text(row.fixed.verified) } } : {}), + ...(row?.matrix?.target && row?.matrix?.check ? { matrix: { target: text(row.matrix.target), check: text(row.matrix.check) } } : {}), + references: (Array.isArray(row?.references) ? row.references : []).map(text).filter(Boolean), + }; +}); + +/** + * What the matrix saw for the check a row points at, so a "worked around" + * claim sits next to this run's evidence: pass, fail, limit, skip, or + * "not run" when no row matches. + */ +export const observedOf = (pointer, rows) => { + if (!pointer) return null; + const row = (Array.isArray(rows) ? rows : []).find((r) => r.target === pointer.target && r.check === pointer.check); + if (!row) return 'not run'; + return isLimitRow(row) ? 'limit' : row.status; +}; + +export const upstreamForResults = (rows, matrixRows) => normaliseUpstream(rows) + .map((row) => (row.matrix ? { ...row, matrix: { ...row.matrix, observed: observedOf(row.matrix, matrixRows) } } : row)); + +/** Validate the file; the rows are normalised when they join a results document. */ +export const loadUpstream = (path) => { + if (!path || !existsSync(path)) return []; + const parsed = JSON.parse(readFileSync(path, 'utf8')); + if (!Array.isArray(parsed)) throw new Error(`${path} must hold an array of upstream rows`); + for (const row of parsed) { + if (!row?.id || !row?.cli || !row?.bug || !row?.workaround?.name) { + throw new Error(`every upstream row needs id, cli, bug and workaround.name: ${JSON.stringify(row)}`); + } + if (!UPSTREAM_STATUSES.includes(row.status)) throw new Error(`upstream row ${row.id} has status ${JSON.stringify(row.status)}; use "worked around", "fixed upstream" or "open"`); + } + return parsed; +}; diff --git a/packages/seats/compat/upstream.json b/packages/seats/compat/upstream.json new file mode 100644 index 0000000000..e3d6ca1292 --- /dev/null +++ b/packages/seats/compat/upstream.json @@ -0,0 +1,126 @@ +[ + { + "id": "goose-refusal-retried", + "cli": "goose", + "versions": "1.54.0 to 1.56.0", + "bug": "retries a 400 content-policy refusal 3 times with a byte-identical body, for both its title call and its reply call", + "status": "worked around", + "workaround": { + "name": "fail fast on the first refusal on the wire, as RunFailure refusal_retry_loop (provider watch in the ACP adapter)", + "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf", + "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js", + "tests": [ + "packages/seats/test/provider-watch.test.js: the first refusal trips once, its body reaches the CLI, and nothing more is forwarded", + "packages/seats/test/acp.test.js: the first provider refusal on the wire ends the run as refusal_retry_loop, fast", + "packages/seats/test/acp-real.test.js: goose acp: the first provider refusal ends the run as refusal_retry_loop" + ] + }, + "matrix": { + "target": "goose@1.56.0", + "check": "refusal" + }, + "references": [] + }, + { + "id": "goose-5xx-silent", + "cli": "goose", + "versions": "1.56.0", + "bug": "over ACP, retries a provider 5xx with backoff and sends no session/update while it does; the error reaches the client only after it gives up, as message text ending in end_turn", + "status": "worked around", + "workaround": { + "name": "our own 5xx detection: the first 5xx on the wire ends the run as RunFailure provider_unavailable (provider watch in the ACP adapter)", + "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf", + "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js", + "tests": [ + "packages/seats/test/provider-watch.test.js: the first 503 trips as provider_unavailable", + "packages/seats/test/acp-real.test.js: goose acp: the first provider 503 ends the run as provider_unavailable" + ] + }, + "matrix": { + "target": "goose@1.56.0", + "check": "provider-5xx" + }, + "references": [] + }, + { + "id": "codex-refusal-exit-0", + "cli": "codex", + "versions": "0.162.1 measured; 0.164.0 reported", + "bug": "an in-band provider refusal ends `codex exec --json` with exit 0 and no output, so the run looks like an empty success", + "status": "open", + "workaround": { + "name": "refusal_dropped detector over the provider response bodies, fed by a per-run loopback provider relay", + "ref": "077a9a12c04429a1ffb6677cb17821368a0c8cef", + "where": "packages/seats/src/wire/refusal.js (the detector, with its unit tests); the per-run relay that feeds it is not built, so it cannot fire on a live run yet", + "tests": [] + }, + "references": [ + "https://github.com/openai/codex/issues/56875" + ] + }, + { + "id": "opencode-5xx-silent", + "cli": "opencode", + "versions": "opencode-ai 1.19.0 to 1.19.4", + "bug": "over ACP, retries a provider 5xx for about 73 s with no session/update, then answers -32603 with no HTTP status", + "status": "worked around", + "workaround": { + "name": "our own 5xx detection: the first 5xx on the wire ends the run as RunFailure provider_unavailable (provider watch in the ACP adapter); pinning @opencode/cli 2.x is the second route, see its row", + "ref": "180b2ec808d87d019a6128e99a4d1196bdc16abf", + "where": "packages/seats/src/provider-watch.js, packages/seats/src/adapters/acp.js", + "tests": [ + "packages/seats/test/provider-watch.test.js: the first 503 trips as provider_unavailable", + "packages/seats/test/acp.test.js: the first provider 5xx on the wire ends the run as provider_unavailable, fast", + "packages/seats/test/acp-real.test.js: opencode acp: the first provider 503 ends the run as provider_unavailable, not at a timeout" + ] + }, + "matrix": { + "target": "opencode@1.19.4", + "check": "provider-5xx" + }, + "references": [ + "https://github.com/anomalyco/opencode/issues/50743", + "https://github.com/anomalyco/opencode/pull/50752" + ] + }, + { + "id": "opencode-v2-retry-status-unread", + "cli": "opencode-v2", + "versions": "@opencode/cli 2.1.4", + "bug": "not an upstream bug: 2.x forwards each retry as a session_info_update whose only content is _meta[\"opencode/retry\"], and our ACP adapter drops that update kind, so the trace shows none of it", + "status": "open", + "workaround": { + "name": "read session_info_update._meta[\"opencode/retry\"] in the ACP adapter's update mapping", + "ref": "", + "where": "packages/seats/src/adapters/acp.js (traceUpdate), not started", + "tests": [] + }, + "matrix": { + "target": "opencode-v2@2.1.4", + "check": "retry-status-surfaced" + }, + "references": [] + }, + { + "id": "pi-cowboy-explore-hang", + "cli": "pi-cowboy", + "versions": "0.22.1 and earlier", + "bug": "an explore agent may not write its result file, because its prompt forbids creating files; a foreground call then never settles", + "status": "fixed upstream", + "fixed": { + "version": "0.22.2", + "verified": "6 of 6 fresh-session runs on 0.22.2 settled (subagent done in 12 to 17 s, parent idle in 16 to 21 s), in a clean node:22-bookworm container with pi 1.4.0, herdr 0.9.5 and pi-cowboy as the only plugin; the same repro on 0.22.1 hung about one run in three" + }, + "workaround": { + "name": "settle a run that ended without a report, in our pi-cowboy plugin; kept for 0.22.1 and earlier, since 0.22.2 still settles only on a report", + "ref": "65ccaf5444f9a34d9d5ebfe28217c863305b8ca8", + "where": "pi-cowboy-commonly 0.1.2 on npm (Team-Commonly/pi-cowboy-commonly, src/report-guard.js). Real-model check on pi-cowboy 0.22.1: 6 of 6 runs settled in 11 to 15 s on deepseek-v4-flash (3 wrote a report, 3 got the missing-report notice), none hung", + "tests": [ + "pi-cowboy-commonly test/report-guard.test.js: KEY: an explore subagent that settles without a report gets a missing-report notice" + ] + }, + "references": [ + "https://github.com/electric-dragon-80000v/pi-cowboy/issues/33" + ] + } +] diff --git a/packages/seats/src/adapters/aider.js b/packages/seats/src/adapters/aider.js new file mode 100644 index 0000000000..2bc412a8a6 --- /dev/null +++ b/packages/seats/src/adapters/aider.js @@ -0,0 +1,85 @@ +/** + * Aider, one turn per process: + * aider --model openai/ --openai-api-base --message --no-stream … + * + * Aider reaches the provider through litellm's OpenAI route; the key goes in + * OPENAI_API_KEY, never argv. It has no machine-readable output: stdout is a + * banner, the reply, then a `Tokens: … sent, … received.` line. A provider + * error prints in place of the reply, with no Tokens line, and aider still + * exits 0, so the Tokens line is what says the turn answered. + * + * Aider has no session ids. A session here is a chat-history file under the + * run's HOME, named by an id the adapter makes; a resume restores it with + * --restore-chat-history. + */ +import { randomUUID } from 'node:crypto'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { runFailure } from '../failures.js'; +import { createHeadlessAdapter, failureFromText } from './headless.js'; + +const SESSION_ID = /^[A-Za-z0-9-]{8,64}$/; +const TOKENS_LINE = /^Tokens: [\d.,]+k? sent/; +// What aider prints before the reply; none of it is the agent's answer. +const BANNER_LINE = /^(?:Aider v\d|Model: |Weak model: |Editor model: |Git repo: |Repo-map: |Restored previous conversation history|Added .* to the chat|https:\/\/aider\.chat\/|Warning: |Note: |Cur working dir: |HTTPSConnectionPool\(|Can't initialize prompt toolkit)/; + +export const readAiderLine = (line, state) => { + (state.lines ||= []).push(line); +}; + +/** The reply is what sits between the banner and the last Tokens line. */ +export const settleAider = (state, { code, stderr, credentials }) => { + const lines = state.lines || []; + let end = -1; + for (let i = lines.length - 1; i >= 0; i -= 1) if (TOKENS_LINE.test(lines[i])) { end = i; break; } + let start = 0; + for (let i = 0; i < (end === -1 ? lines.length : end); i += 1) if (BANNER_LINE.test(lines[i])) start = i + 1; + if (end === -1) { + const said = lines.slice(start).filter((l) => l.trim() && !/^Retrying in /.test(l)).join(' '); + throw failureFromText('aider', said || stderr || `exit ${code} with no reply`, { credentials, exitCode: code }); + } + if (code !== 0) throw failureFromText('aider', stderr || `exit ${code}`, { credentials, exitCode: code }); + const text = lines.slice(start, end).join('\n').trim(); + if (!text) throw runFailure('protocol', 'aider printed a Tokens line and no reply'); + return { text, newSessionId: state.sessionId }; +}; + +export const createAiderAdapter = createHeadlessAdapter({ + name: 'aider', + cli: 'aider', + command: 'aider', + versionPattern: /aider\s+v?(\d+\.\d+\.\d+[^\s]*)/, + async launch(prompt, ctx, { provider, env, home }) { + if (ctx.sessionId && !SESSION_ID.test(ctx.sessionId)) throw runFailure('unsupported', 'aider: session id is not one this adapter made'); + const sessionId = ctx.sessionId || randomUUID(); + const dir = join(home, '.seats-aider'); + await mkdir(dir, { recursive: true, mode: 0o700 }); + return { + args: [ + '--model', `openai/${provider.model}`, '--openai-api-base', provider.baseURL, + '--message', prompt, + '--chat-history-file', join(dir, `${sessionId}.chat.md`), + '--input-history-file', join(dir, `${sessionId}.input`), + ...(ctx.sessionId ? ['--restore-chat-history'] : ['--no-restore-chat-history']), + '--llm-history-file', '/dev/null', + '--yes-always', '--no-git', '--no-gitignore', '--no-auto-commits', '--no-auto-lint', + '--no-pretty', '--no-stream', '--no-fancy-input', '--no-detect-urls', '--no-suggest-shell-commands', + '--no-check-update', '--no-analytics', '--no-show-model-warnings', '--no-show-release-notes', + '--map-tokens', '0', + ], + env: { + ...env, + OPENAI_API_KEY: provider.apiKey, + // litellm would otherwise fetch its model price list from GitHub on start. + LITELLM_LOCAL_MODEL_COST_MAP: 'True', + AIDER_ANALYTICS: 'false', + AIDER_CHECK_UPDATE: 'false', + }, + state: { sessionId }, + }; + }, + read: readAiderLine, + settle: settleAider, +}); + +export default createAiderAdapter; diff --git a/packages/seats/src/adapters/cline.js b/packages/seats/src/adapters/cline.js new file mode 100644 index 0000000000..7e3cb44384 --- /dev/null +++ b/packages/seats/src/adapters/cline.js @@ -0,0 +1,106 @@ +/** + * Cline CLI, one headless turn per process: + * cline --json --config -P openai-compatible -m [--id ] + * + * Cline takes a provider only from its settings store (`cline auth` wants the + * key in argv), so the adapter writes /data/settings/providers.json + * itself, mode 0600, with the run's base URL and key. The config directory + * lives under the run's HOME, so a resume finds the task the first turn made. + * The prompt is the one positional argument; the key is never in argv. + * + * stdout is NDJSON: agent_event content_start carries text, and run_result + * closes the turn with finishReason and text. A failure also arrives on + * stderr as {"type":"error","message":...}. + * + * The id `--id` resumes is the session id `cline history` lists, which + * stdout never prints (its taskId is a different id). Each turn records one + * directory under /data/sessions/ named by it, so the session id is + * the one directory the turn added. Measured on 3.0.73: `--id` with `--json` + * then refuses every prompt form (positional, after `--`, piped stdin) with + * "JSON output mode requires a prompt argument or piped stdin", so a resume + * fails by name; targets.json carries that as a limit. + */ +import { readdirSync } from 'node:fs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { createHeadlessAdapter, failureFromText } from './headless.js'; +import { runFailure } from '../failures.js'; + +const PROVIDER = 'openai-compatible'; + +const parse = (line) => { + try { return JSON.parse(line); } catch { return null; } +}; + +export const readClineEvent = (line, state) => { + const event = parse(line); + if (!event || typeof event !== 'object') return; + if (event.type === 'agent_event') { + const inner = event.event || {}; + if (inner.type === 'content_start' && inner.contentType === 'text' && typeof inner.text === 'string' && inner.text) state.chunks.push(inner.text); + if (inner.type === 'error') state.lastError = String(inner.error?.message || 'agent error'); + } else if (event.type === 'run_result') { + if (event.finishReason === 'completed') state.text = String(event.text ?? ''); + else state.error = String(event.text || state.lastError || `cline finished with ${event.finishReason}`); + } else if (event.type === 'error' && !state.error) { + state.error = String(event.message || 'cline error'); + } +}; + +// Cline ignores a provider entry with no updatedAt and falls back to its own +// default route (api.openai.com, or api.cline.bot with no -P), measured on +// 3.0.73: the file `cline auth` writes differs from this one only by it. +export const providersFile = ({ baseURL, apiKey, model }, now = new Date()) => ({ + version: 1, + lastUsedProvider: PROVIDER, + modes: {}, + providers: { + [PROVIDER]: { + settings: { provider: PROVIDER, apiKey, model, baseUrl: baseURL }, + updatedAt: now.toISOString(), + tokenSource: 'manual', + }, + }, +}); + +const sessionsOf = (dir) => { + try { return readdirSync(dir); } catch { return []; } +}; + +export const settleCline = (state, { code, stderr, credentials }) => { + if (state.error || code !== 0) { + throw failureFromText('cline', state.error || stderr || `exit ${code}`, { credentials, exitCode: code }); + } + if (typeof state.text !== 'string') throw runFailure('protocol', 'cline exited 0 without a run_result'); + const before = new Set(state.sessionsBefore || []); + const added = state.sessionsDir ? sessionsOf(state.sessionsDir).filter((name) => !before.has(name)) : []; + const sessionId = state.resumed || (added.length === 1 ? added[0] : null); + return { text: state.text, ...(sessionId ? { newSessionId: sessionId } : {}) }; +}; + +export const createClineAdapter = createHeadlessAdapter({ + name: 'cline', + cli: 'cline', + command: 'cline', + versionPattern: /(\d+\.\d+\.\d+[^\s]*)/, + async launch(prompt, ctx, { provider, env, home }) { + const config = join(home, '.cline'); + const settings = join(config, 'data', 'settings'); + await mkdir(settings, { recursive: true, mode: 0o700 }); + await writeFile(join(settings, 'providers.json'), JSON.stringify(providersFile(provider), null, 2), { mode: 0o600 }); + const sessionsDir = join(config, 'data', 'sessions'); + return { + state: { sessionsDir, sessionsBefore: sessionsOf(sessionsDir), resumed: ctx.sessionId || null }, + args: [ + '--json', '--config', config, '-P', PROVIDER, '-m', provider.model, + ...(ctx.sessionId ? ['--id', ctx.sessionId] : []), + '--', prompt, + ], + env, + }; + }, + read: readClineEvent, + settle: settleCline, +}); + +export default createClineAdapter; diff --git a/packages/seats/src/adapters/copilot.js b/packages/seats/src/adapters/copilot.js new file mode 100644 index 0000000000..f38a56729e --- /dev/null +++ b/packages/seats/src/adapters/copilot.js @@ -0,0 +1,71 @@ +/** + * GitHub Copilot CLI, one headless turn per process: + * copilot --output-format json --allow-all-tools [--resume ] < prompt + * + * Copilot reaches the run's provider through its BYOK variables + * (COPILOT_PROVIDER_BASE_URL with type openai). With COPILOT_OFFLINE set it + * makes no GitHub call at all, so no GitHub sign-in is needed. The key goes + * in COPILOT_PROVIDER_API_KEY, never argv; the prompt goes on stdin. + * + * stdout is JSONL: assistant.message_delta / assistant.message carry the + * reply, session.error carries a provider failure with its HTTP status, and + * the closing `result` event carries the session id a resume names. + */ +import { join } from 'node:path'; +import { createHeadlessAdapter } from './headless.js'; + +const parse = (line) => { + try { return JSON.parse(line); } catch { return null; } +}; + +export const readCopilotEvent = (line, state) => { + const event = parse(line); + if (!event || typeof event !== 'object') return; + const data = event.data || {}; + switch (event.type) { + case 'assistant.message_delta': + if (typeof data.deltaContent === 'string' && data.deltaContent) state.chunks.push(data.deltaContent); + break; + case 'assistant.message': + // A turn with tool calls has several assistant messages; the reply is + // the last one that says something. + if (typeof data.content === 'string' && data.content) state.text = data.content; + break; + case 'session.error': + state.error = String(data.message || data.errorType || 'session.error'); + break; + case 'result': + if (typeof event.sessionId === 'string' && event.sessionId) state.sessionId = event.sessionId; + break; + default: + } +}; + +export const createCopilotAdapter = createHeadlessAdapter({ + name: 'copilot', + cli: 'copilot', + command: 'copilot', + versionPattern: /Copilot CLI\s+v?(\d+\.\d+\.\d+[^\s.]*)/, + async launch(prompt, ctx, { provider, env, home }) { + return { + args: [ + '--output-format', 'json', '--allow-all-tools', + ...(ctx.sessionId ? [`--resume=${ctx.sessionId}`] : []), + ], + stdin: prompt, + env: { + ...env, + COPILOT_HOME: join(home, '.copilot'), + COPILOT_OFFLINE: 'true', + COPILOT_AUTO_UPDATE: 'false', + COPILOT_PROVIDER_TYPE: 'openai', + COPILOT_PROVIDER_BASE_URL: provider.baseURL, + COPILOT_PROVIDER_API_KEY: provider.apiKey, + COPILOT_MODEL: provider.model, + }, + }; + }, + read: readCopilotEvent, +}); + +export default createCopilotAdapter; diff --git a/packages/seats/src/adapters/headless.js b/packages/seats/src/adapters/headless.js new file mode 100644 index 0000000000..25e960a6a6 --- /dev/null +++ b/packages/seats/src/adapters/headless.js @@ -0,0 +1,269 @@ +/** + * One adapter shape for agent CLIs that run one turn per process in their own + * headless mode (`copilot -p`, `qwen -o stream-json`, `aider --message`, …). + * The CLI-specific part is a `launch(prompt, ctx)` that returns argv and env, + * a `read(event, state)` for each stdout line, and a `settle(state)` that + * turns what was read into a reply. Everything else is here, once: the + * provider route check, an isolated HOME, the process group, the turn + * deadline, cancellation, the trace and the scrubbed failure text. + * + * The provider key travels in the child env or a 0600 file, never in argv, + * which any same-user process can read. + */ +import { spawn as childSpawn, spawnSync } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { StringDecoder } from 'node:string_decoder'; +import { RunFailure, runFailure } from '../failures.js'; +import { observeSpawn } from '../runtime-observation.js'; +import { traceOf } from '../trace/events.js'; +import { scrubAdapterFailure } from '../upstream-refusal.js'; + +const SAFE_ENV = /^(?:PATH|LANG|LC_[A-Z_]+|TERM|USER|LOGNAME|NO_COLOR|CI|TMPDIR|SSL_CERT_FILE|SSL_CERT_DIR)$/; +const VERSION_TIMEOUT_MS = 15000; +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; +const KILL_GRACE_MS = 2000; +const MAX_STDERR_CHARS = 8192; +const MAX_LINE_CHARS = 1024 * 1024; +const FAILURE_LIMIT = 500; +const REFUSAL_WORDING = /content[ _-]?policy|refus|rejected by/i; +const OWN_GROUP = process.platform !== 'win32'; + +/** The provider route, checked: a plain HTTP(S) base URL, a key and a model. */ +export const providerOf = (ctx, label) => { + const { baseURL, apiKey, model } = ctx?.provider || {}; + let url; + try { url = new URL(String(baseURL)); } catch { + throw runFailure('unsupported', `${label}: ctx.provider.baseURL must be an HTTP(S) URL`); + } + if (!/^https?:$/.test(url.protocol) || url.username || url.password || url.search || url.hash) { + throw runFailure('unsupported', `${label}: ctx.provider.baseURL must be a plain HTTP(S) URL with no credentials, query or fragment`); + } + if (typeof apiKey !== 'string' || !apiKey) throw runFailure('unsupported', `${label}: ctx.provider.apiKey is missing`); + if (typeof model !== 'string' || !model) throw runFailure('unsupported', `${label}: ctx.provider.model is missing`); + return { url, baseURL: url.href.replace(/\/$/, ''), apiKey, model }; +}; + +/** + * Safe launch variables plus HOME and XDG roots. HOME comes from ctx.env when + * the caller gives one, because a resume reads the sessions kept there; + * without one the spawn gets a throwaway HOME that is removed afterwards. + */ +export const isolatedEnv = async (ctx, label) => { + const source = ctx.env || process.env; + const env = Object.fromEntries(Object.entries(source).filter(([name]) => SAFE_ENV.test(name))); + const callerHome = ctx.env?.HOME; + const home = callerHome || await mkdtemp(join(tmpdir(), `seats-${label}-home-`)); + env.HOME = home; + env.XDG_CONFIG_HOME = ctx.env?.XDG_CONFIG_HOME || join(home, '.config'); + env.XDG_DATA_HOME = ctx.env?.XDG_DATA_HOME || join(home, '.local', 'share'); + env.XDG_STATE_HOME = ctx.env?.XDG_STATE_HOME || join(home, '.local', 'state'); + env.XDG_CACHE_HOME = ctx.env?.XDG_CACHE_HOME || join(home, '.cache'); + env.NO_COLOR = '1'; + const removeHome = callerHome ? async () => {} : () => rm(home, { recursive: true, force: true }); + return { env, home, removeHome }; +}; + +/** ` --version`, read with `pattern`; null when the binary is absent or fails. */ +export const readVersion = (binary, pattern, { args = ['--version'], env = process.env, spawnSyncImpl = spawnSync } = {}) => { + const probe = spawnSyncImpl(binary, args, { encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS }); + if (probe.error || probe.status !== 0) return null; + const version = pattern.exec(`${probe.stdout || ''}\n${probe.stderr || ''}`)?.[1]; + return version ? { path: binary, version } : null; +}; + +/** + * A CLI's own failure text, as a named failure: a 4xx or refusal wording is a + * refusal (never retried), anything else is the process ending unsettled. + */ +export const failureFromText = (label, text, { credentials = [], exitCode = null } = {}) => { + const raw = String(text ?? '').trim(); + const { status, detail } = scrubAdapterFailure(raw, { credentials, limit: FAILURE_LIMIT }); + const refusal = (status !== null && status >= 400 && status <= 499) || REFUSAL_WORDING.test(raw); + const code = exitCode === null ? '' : ` (exit ${exitCode})`; + const upstream = status === null ? '' : ` upstream ${status}`; + return runFailure(refusal ? 'refusal' : 'exited', `${label}${code}${upstream}: ${detail || 'no error text'}`, { + credentials, limit: FAILURE_LIMIT, + }); +}; + +const signal = (child, sig) => { + if (OWN_GROUP && child.pid !== undefined) { + try { process.kill(-child.pid, sig); return; } catch { /* group gone; fall through */ } + } + try { child.kill(sig); } catch { /* already gone */ } +}; + +/** + * @param {object} spec + * @param {string} spec.name adapter name + * @param {string} spec.cli CLI name in traces and failures + * @param {string} spec.command the command when no binaryPath is given + * @param {RegExp} spec.versionPattern reads the version out of `--version` + * @param {string[]} [spec.versionArgs] + * @param {(prompt: string, ctx: object, run: { command: string, provider: object, env: object, home: string }) => + * Promise<{ args: string[], env?: object, stdin?: string, state?: object, cleanup?: () => Promise }>} spec.launch + * (`state` seeds what read() and settle() see, e.g. a session id the adapter made) + * @param {(line: string, state: object) => void} spec.read one stdout line; may set state.text, + * state.sessionId, state.error, state.chunks (pushed text, traced as agent.message_chunk) + * @param {(state: object, end: { code: number|null, stderr: string, credentials: string[] }) => + * { text: string, newSessionId?: string }} [spec.settle] + */ +export const createHeadlessAdapter = (spec) => ({ binaryPath } = {}) => { + const command = binaryPath || spec.command; + const versionArgs = spec.versionArgs || ['--version']; + const settle = spec.settle || ((state, { code, stderr, credentials }) => { + if (state.error || code !== 0) { + throw failureFromText(spec.cli, state.error || stderr || `exit ${code}`, { credentials, exitCode: code }); + } + if (typeof state.text !== 'string') throw runFailure('protocol', `${spec.cli} exited 0 without a reply`); + return { text: state.text, ...(state.sessionId ? { newSessionId: state.sessionId } : {}) }; + }); + + return { + name: spec.name, + runtimeType: spec.name, + + async detect() { + return readVersion(command, spec.versionPattern, { args: versionArgs }); + }, + + async spawn(prompt, ctx = {}) { + const trace = traceOf(ctx); + const started = Date.now(); + const settled = (outcome, failure) => trace.emit('run.settled', { + outcome, durationMs: Date.now() - started, attempts: 1, + ...(failure ? { failure: { kind: failure.kind, detail: failure.detail } } : {}), + }); + trace.emit('run.started', { adapter: spec.name, cli: { name: spec.cli }, sessionId: ctx.sessionId, attempt: 1 }); + let launched; + let removeHome = async () => {}; + try { + const provider = providerOf(ctx, spec.cli); + const isolated = await isolatedEnv(ctx, spec.cli); + removeHome = isolated.removeHome; + launched = await spec.launch(String(prompt ?? ''), ctx, { command, provider, env: isolated.env, home: isolated.home }); + } catch (error) { + await removeHome(); + const failure = error instanceof RunFailure ? error : runFailure('unsupported', `${spec.cli}: ${error?.message || error}`); + settled('failed', failure); + throw failure; + } + const credentials = [ctx.provider?.apiKey, ctx.runtimeToken].filter((v) => typeof v === 'string' && v.length >= 8); + const requested = Number(ctx.timeoutMs); + const timeoutMs = Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_TIMEOUT_MS; + trace.emit('turn.prompt_sent', { chars: String(prompt ?? '').length }); + try { + const end = await run({ + command, launched, ctx, timeoutMs, read: spec.read, trace, + }); + if (end.ended === 'aborted') { + throw ctx.signal?.reason instanceof RunFailure ? ctx.signal.reason : runFailure('cancelled', 'the caller cancelled the turn'); + } + if (end.ended === 'timeout') throw runFailure('timeout', `${spec.cli} made no answer within ${timeoutMs}ms`); + if (end.ended === 'spawn-error') throw runFailure('exited', `${spec.cli} could not be started: ${end.spawnError}`, { credentials }); + if (end.ended === 'protocol') throw runFailure('protocol', `${spec.cli}: ${end.protocolError}`, { credentials }); + const result = settle(end.state, { code: end.code, stderr: end.stderr, credentials }); + settled('ok'); + return result; + } catch (error) { + const failure = error instanceof RunFailure ? error : runFailure('protocol', `${spec.cli}: ${error?.message || error}`, { credentials }); + settled(failure.kind === 'cancelled' ? 'cancelled' : 'failed', failure); + throw failure; + } finally { + await launched.cleanup?.().catch(() => {}); + await removeHome().catch(() => {}); + } + }, + }; +}; + +const run = ({ command, launched, ctx, timeoutMs, read, trace }) => new Promise((resolve) => { + const spawnImpl = ctx._spawnImpl || childSpawn; + const child = spawnImpl(command, launched.args, { + cwd: ctx.cwd, env: launched.env, stdio: ['pipe', 'pipe', 'pipe'], detached: OWN_GROUP, + }); + observeSpawn(child, ctx.onSpawn, launched.controls || {}); + const state = { ...(launched.state || {}), chunks: [] }; + const decoder = new StringDecoder('utf8'); + let buffer = ''; + let stderr = ''; + let ended = null; + let protocolError = null; + let spawnError = null; + let killTimer; + let closed = false; + const stop = (why) => { + if (ended || closed) return; + ended = why; + signal(child, 'SIGTERM'); + killTimer = setTimeout(() => { if (!closed) signal(child, 'SIGKILL'); }, KILL_GRACE_MS); + killTimer.unref?.(); + }; + const timer = setTimeout(() => stop('timeout'), timeoutMs); + const onAbort = () => stop('aborted'); + ctx.signal?.addEventListener('abort', onAbort, { once: true }); + if (ctx.signal?.aborted) onAbort(); + const take = (line) => { + if (!line || ended) return; + const before = state.chunks.length; + try { read(line, state); } catch (error) { + protocolError = error?.message || String(error); + stop('protocol'); + return; + } + for (const text of state.chunks.slice(before)) trace.emit('agent.message_chunk', { text }); + }; + child.stdout.on('data', (chunk) => { + buffer += decoder.write(chunk); + let at; + while ((at = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, at).replace(/\r$/, ''); + buffer = buffer.slice(at + 1); + take(line); + } + if (buffer.length > MAX_LINE_CHARS) { + protocolError = 'a stdout line exceeded 1 MiB'; + stop('protocol'); + } + }); + child.stderr.on('data', (chunk) => { stderr = `${stderr}${chunk}`.slice(-MAX_STDERR_CHARS); }); + child.on('error', (error) => { spawnError = error?.code || error?.message || 'spawn failed'; stop('spawn-error'); }); + child.on('close', (code, sig) => { + closed = true; + clearTimeout(timer); + clearTimeout(killTimer); + ctx.signal?.removeEventListener('abort', onAbort); + take(`${buffer}${decoder.end()}`.trim()); + // Whatever the CLI started in its group (MCP servers, tool shells) goes too. + if (OWN_GROUP && child.pid !== undefined) { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* group gone */ } } + trace.emit('process.exited', { code, signal: sig }); + resolve({ state, code, stderr, ended, protocolError, spawnError }); + }); + child.stdin.on('error', () => {}); + child.stdin.end(launched.stdin ?? ''); +}); + +/** + * An adapter for a CLI that cannot be pointed at the run's provider. detect() + * still reads the pinned binary, so the matrix proves it is installed; every + * turn fails as RunFailure provider_unsupported with `detail`, before any + * process starts, so a check reads a named reason rather than a hang. + */ +export const createProviderUnsupportedAdapter = ({ name, cli, command, versionPattern, versionArgs, detail }) => ({ binaryPath } = {}) => { + const bin = binaryPath || command; + return { + name, + runtimeType: name, + supportsProvider: false, + async detect() { return readVersion(bin, versionPattern, { args: versionArgs || ['--version'] }); }, + async spawn(_prompt, ctx = {}) { + const trace = traceOf(ctx); + const failure = runFailure('provider_unsupported', `${cli}: ${detail}`); + trace.emit('run.started', { adapter: name, cli: { name: cli }, attempt: 1 }); + trace.emit('run.settled', { outcome: 'failed', failure: { kind: failure.kind, detail: failure.detail }, durationMs: 0, attempts: 1 }); + throw failure; + }, + }; +}; diff --git a/packages/seats/src/adapters/provider-unsupported.js b/packages/seats/src/adapters/provider-unsupported.js new file mode 100644 index 0000000000..9016f08af0 --- /dev/null +++ b/packages/seats/src/adapters/provider-unsupported.js @@ -0,0 +1,36 @@ +/** + * CLIs on the compat matrix that cannot be pointed at the run's provider. + * Each detects its pinned binary like any other target, and every turn fails + * as RunFailure provider_unsupported naming why, so the matrix shows a named + * red row rather than a skip. The reasons were read from each CLI's own + * --help and docs at the pinned version; a release that adds a custom + * OpenAI- or Anthropic-compatible base URL should get a real adapter. + */ +import { createProviderUnsupportedAdapter } from './headless.js'; + +export const createGeminiAdapter = createProviderUnsupportedAdapter({ + name: 'gemini', + cli: 'gemini', + command: 'gemini', + versionPattern: /(\d+\.\d+\.\d+[^\s]*)/, + detail: 'Gemini CLI speaks only the Gemini API (GOOGLE_GEMINI_BASE_URL and Vertex both expect a Gemini endpoint); ' + + 'it has no OpenAI- or Anthropic-compatible provider route', +}); + +export const createCursorAdapter = createProviderUnsupportedAdapter({ + name: 'cursor', + cli: 'cursor-agent', + command: 'cursor-agent', + versionPattern: /(\d{4}\.\d+\.\d+-[0-9a-f]+)/, + detail: "cursor-agent sends every model call through Cursor's own backend (--endpoint names Cursor's API, not a model provider) " + + 'and signs in with CURSOR_API_KEY; it has no OpenAI- or Anthropic-compatible base URL', +}); + +export const createAmpAdapter = createProviderUnsupportedAdapter({ + name: 'amp', + cli: 'amp', + command: 'amp', + versionPattern: /(\d+\.\d+\.\d+-g[0-9a-f]+)/, + detail: 'Amp sends every model call through the Amp service (AMP_URL names an Amp server, not a model provider) ' + + 'and signs in with AMP_API_KEY; it has no OpenAI- or Anthropic-compatible base URL', +}); diff --git a/packages/seats/src/adapters/qwen.js b/packages/seats/src/adapters/qwen.js new file mode 100644 index 0000000000..373038739d --- /dev/null +++ b/packages/seats/src/adapters/qwen.js @@ -0,0 +1,70 @@ +/** + * Qwen Code, one headless turn per process: + * qwen --bare --safe-mode --auth-type openai -m -o stream-json [--resume ] < prompt + * + * The same invocation the CLI's production qwen adapter uses. The provider + * goes in OPENAI_BASE_URL / OPENAI_API_KEY; QWEN_HOME is the run's HOME so a + * resume finds the session the first turn recorded. + * + * stdout is stream-json: `system/init` and every later event carry + * session_id; `assistant` carries text blocks; `result` closes the turn and + * says is_error, with the provider's text in error.message. + */ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { createHeadlessAdapter } from './headless.js'; + +const parse = (line) => { + try { return JSON.parse(line); } catch { return null; } +}; + +export const readQwenEvent = (line, state) => { + const event = parse(line); + if (!event || typeof event !== 'object') return; + if (typeof event.session_id === 'string' && event.session_id) state.sessionId = event.session_id; + if (event.type === 'assistant') { + const text = (event.message?.content || []) + .filter((block) => block?.type === 'text' && typeof block.text === 'string') + .map((block) => block.text).join(''); + if (text) state.chunks.push(text); + } else if (event.type === 'result') { + if (event.is_error) state.error = String(event.error?.message || event.result || `qwen result ${event.subtype || 'error'}`); + else if (typeof event.result === 'string') state.text = event.result; + } +}; + +export const createQwenAdapter = createHeadlessAdapter({ + name: 'qwen', + cli: 'qwen', + command: 'qwen', + versionPattern: /(\d+\.\d+\.\d+[^\s]*)/, + async launch(prompt, ctx, { provider, env, home }) { + const qwenHome = join(home, '.qwen'); + await mkdir(qwenHome, { recursive: true, mode: 0o700 }); + const settingsPath = join(qwenHome, 'system-settings.json'); + await writeFile(settingsPath, JSON.stringify({ + privacy: { usageStatisticsEnabled: false }, + general: { enableAutoUpdate: false }, + }), { mode: 0o600 }); + return { + args: [ + '--bare', '--safe-mode', '--auth-type', 'openai', '-m', provider.model, '-o', 'stream-json', + ...(ctx.sessionId ? ['--resume', ctx.sessionId] : []), + ], + stdin: prompt, + env: { + ...env, + QWEN_HOME: qwenHome, + QWEN_RUNTIME_DIR: qwenHome, + QWEN_CODE_SYSTEM_SETTINGS_PATH: settingsPath, + QWEN_USAGE_STATISTICS_ENABLED: 'false', + OPENAI_BASE_URL: provider.baseURL, + OPENAI_API_KEY: provider.apiKey, + OPENAI_MODEL: provider.model, + }, + }; + }, + read: readQwenEvent, +}); + +export default createQwenAdapter; diff --git a/packages/seats/src/failures.js b/packages/seats/src/failures.js index b3ff12fb1d..36a839b660 100644 --- a/packages/seats/src/failures.js +++ b/packages/seats/src/failures.js @@ -10,6 +10,7 @@ export const FAILURE_KINDS = Object.freeze({ timeout: { retryable: true, meaning: 'the run exceeded its wall-clock budget' }, exited: { retryable: true, meaning: 'the CLI process ended before the turn settled' }, protocol: { retryable: false, meaning: 'the CLI broke its wire protocol' }, + provider_unsupported: { retryable: false, meaning: 'the CLI cannot be pointed at the run\'s OpenAI- or Anthropic-compatible provider' }, refusal: { retryable: false, meaning: 'the model or provider refused; never retried' }, max_tokens: { retryable: false, meaning: 'the turn hit its token limit' }, max_turn_requests: { retryable: false, meaning: 'the turn hit its request limit' }, diff --git a/packages/seats/test-support/fake-cli-adapter.js b/packages/seats/test-support/fake-cli-adapter.js new file mode 100644 index 0000000000..6e136722dc --- /dev/null +++ b/packages/seats/test-support/fake-cli-adapter.js @@ -0,0 +1,149 @@ +/** + * An adapter for test-support/fake-cli.mjs. It exists so the compatibility + * runner is proven against a real child process — argv, env, stdio, exit codes + * and kills — before any third-party CLI is wired in. + * + * `createFakeCliAdapter({ fault })` builds a deliberately broken variant, one + * defect per name, so each check can be shown to fail for the reason it names: + * adapter-side: key-in-argv, retry-refusal, no-timeout, plain-error, + * unscrubbed-error, bad-trace + * child-side (passed through FAKE_CLI_FAULT): echo-key, extra-text, + * new-session, error-echo-key, double-call, retry-5xx-forever + * version: { version } overrides what the child reports + */ +import { spawn as childSpawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { StringDecoder } from 'node:string_decoder'; +import { RunFailure, runFailure } from '../src/failures.js'; +import { traceOf } from '../src/trace/events.js'; +import { observeSpawn } from '../src/runtime-observation.js'; + +export const FAKE_CLI_PATH = fileURLToPath(new URL('./fake-cli.mjs', import.meta.url)); +const CHILD_FAULTS = new Set(['echo-key', 'extra-text', 'new-session', 'error-echo-key', 'double-call', 'retry-5xx-forever']); +const DEFAULT_TIMEOUT_MS = 30000; +const KILL_GRACE_MS = 2000; + +// A provider body that names a content policy is a refusal; anything else +// from the provider is an upstream failure the CLI could not get past. +const failureFor = (result, credentials, scrub) => { + const status = Number(result?.status) || 0; + const text = `provider answered ${status}: ${String(result?.error || '').slice(0, 300)}`; + const kind = status === 400 && /content.policy|content_policy/i.test(String(result?.error)) ? 'refusal' : 'exited'; + return scrub ? runFailure(kind, text, { credentials }) : new RunFailure(kind, text); +}; + +const runOnce = ({ args, prompt, env, cwd, timeoutMs, signal, onSpawn, trace }) => new Promise((resolve) => { + const child = childSpawn(process.execPath, args, { cwd, env, stdio: ['pipe', 'pipe', 'pipe'] }); + observeSpawn(child, onSpawn, { model: env.FAKE_CLI_MODEL }); + const decoder = new StringDecoder('utf8'); + let buffer = ''; + let result = null; + let ended = null; // why we stopped the child, if we did + let killTimer; + const stop = (reason) => { + if (ended) return; + ended = reason; + child.kill('SIGTERM'); + killTimer = setTimeout(() => child.kill('SIGKILL'), KILL_GRACE_MS); + }; + const timer = timeoutMs ? setTimeout(() => stop('timeout'), timeoutMs) : null; + const onAbort = () => stop('aborted'); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) onAbort(); + child.stdout.on('data', (chunk) => { + buffer += decoder.write(chunk); + let newline; + while ((newline = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line) continue; + let event; + try { event = JSON.parse(line); } catch { continue; } + if (event.type === 'message') trace.emit('agent.message_chunk', { text: String(event.text ?? '') }); + if (event.type === 'result') result = event; + } + }); + child.stderr.resume(); + child.on('error', () => stop('spawn-error')); + child.on('close', (code, sig) => { + clearTimeout(timer); + clearTimeout(killTimer); + signal?.removeEventListener('abort', onAbort); + trace.emit('process.exited', { code, signal: sig }); + resolve({ result, code, ended }); + }); + child.stdin.on('error', () => {}); + child.stdin.end(prompt); +}); + +export const createFakeCliAdapter = ({ fault = null, version = null, bin = FAKE_CLI_PATH } = {}) => ({ + name: fault ? `fake-cli:${fault}` : 'fake-cli', + runtimeType: 'fake-cli', + providerKeyEnv: 'FAKE_CLI_API_KEY', + + async detect() { + const env = { ...process.env, ...(version ? { FAKE_CLI_VERSION: version } : {}) }; + const probe = spawnSync(process.execPath, [bin, '--version'], { encoding: 'utf8', env, timeout: 10000 }); + const found = /^fake-cli (\S+)/.exec(String(probe.stdout || ''))?.[1]; + return probe.status === 0 && found ? { path: bin, version: found } : null; + }, + + async spawn(prompt, ctx = {}) { + const trace = traceOf(ctx); + const provider = ctx.provider; + if (!provider?.baseURL) throw runFailure('unsupported', 'fake-cli needs ctx.provider'); + const credentials = provider.apiKey ? [provider.apiKey] : []; + const env = { + ...(ctx.env || {}), + FAKE_CLI_BASE_URL: provider.baseURL, + FAKE_CLI_MODEL: provider.model || '', + FAKE_CLI_API_KEY: provider.apiKey || '', + ...(CHILD_FAULTS.has(fault) ? { FAKE_CLI_FAULT: fault } : {}), + }; + const args = [bin, 'run']; + if (ctx.sessionId) args.push('--resume', ctx.sessionId); + if (fault === 'key-in-argv') args.push('--api-key', provider.apiKey); + const requested = Number(ctx.timeoutMs); + const timeoutMs = fault === 'no-timeout' ? 0 + : (Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_TIMEOUT_MS); + const started = Date.now(); + const settle = (outcome, failure) => { + const data = { outcome, durationMs: Date.now() - started, attempts }; + if (failure) data.failure = { kind: failure.kind, detail: failure.detail }; + trace.emit('run.settled', data); + }; + + let attempts = 0; + const maxAttempts = fault === 'retry-refusal' ? 2 : 1; + if (fault === 'bad-trace') trace.emit('turn.prompt_sent', { chars: prompt.length }); + for (;;) { + attempts += 1; + trace.emit('run.started', { adapter: this.name, cli: { name: 'fake-cli', version: version || '1.0.0' }, sessionId: ctx.sessionId, attempt: attempts }); + if (fault !== 'bad-trace') trace.emit('turn.prompt_sent', { chars: prompt.length }); + const { result, ended } = await runOnce({ + args, prompt, env, cwd: ctx.cwd, timeoutMs, signal: ctx.signal, onSpawn: ctx.onSpawn, trace, + }); + let failure = null; + if (ended === 'aborted') { + const reason = ctx.signal?.reason; + failure = reason instanceof RunFailure ? reason : runFailure('cancelled', 'the caller cancelled the turn'); + } else if (ended === 'timeout') { + failure = runFailure('timeout', `fake-cli made no answer within ${timeoutMs}ms`); + } else if (ended === 'spawn-error' || !result) { + failure = runFailure('exited', 'fake-cli ended without a result'); + } else if (!result.ok) { + failure = failureFor(result, credentials, fault !== 'unscrubbed-error' && fault !== 'error-echo-key'); + } + if (!failure) { + settle('ok'); + return { text: String(result.text ?? ''), newSessionId: result.session_id }; + } + if (fault === 'retry-refusal' && failure.kind === 'refusal' && attempts < maxAttempts) continue; + settle(failure.kind === 'cancelled' ? 'cancelled' : 'failed', failure); + if (fault === 'plain-error') throw new Error(failure.detail); + throw failure; + } + }, +}); + +export default createFakeCliAdapter(); diff --git a/packages/seats/test-support/fake-cli.mjs b/packages/seats/test-support/fake-cli.mjs new file mode 100644 index 0000000000..d9173de24e --- /dev/null +++ b/packages/seats/test-support/fake-cli.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/** + * A tiny agent CLI for the compatibility matrix: a real child process that + * reads a prompt on stdin, makes one OpenAI-compatible call, and writes JSONL. + * + * fake-cli --version prints "fake-cli " + * fake-cli run [--resume ID] one turn; the prompt arrives on stdin + * + * Provider settings arrive in env (FAKE_CLI_BASE_URL, FAKE_CLI_MODEL, + * FAKE_CLI_API_KEY), or the key as `--api-key K` when an adapter misbehaves. + * FAKE_CLI_FAULT selects a child-side defect so the matrix can prove it + * notices one: echo-key, extra-text, new-session, error-echo-key, double-call, + * retry-5xx-forever. + * + * Output lines: {type:'message', text} then {type:'result', ok:true, text, + * session_id} or {type:'result', ok:false, status, error}. + */ +import { randomUUID } from 'node:crypto'; + +const VERSION = process.env.FAKE_CLI_VERSION || '1.0.0'; +const argv = process.argv.slice(2); +const flag = (name) => { + const at = argv.indexOf(name); + return at === -1 ? null : argv[at + 1] ?? null; +}; +const write = (event) => process.stdout.write(`${JSON.stringify(event)}\n`); + +if (argv[0] === '--version') { + process.stdout.write(`fake-cli ${VERSION}\n`); + process.exit(0); +} +if (argv[0] !== 'run') { + process.stderr.write('usage: fake-cli run [--resume ID]\n'); + process.exit(2); +} + +const fault = process.env.FAKE_CLI_FAULT || ''; +const key = flag('--api-key') || process.env.FAKE_CLI_API_KEY || ''; +const sessionId = (fault !== 'new-session' && flag('--resume')) || `fake-${randomUUID()}`; + +let prompt = ''; +process.stdin.setEncoding('utf8'); +for await (const chunk of process.stdin) prompt += chunk; + +const ask = () => fetch(`${process.env.FAKE_CLI_BASE_URL}/chat/completions`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ model: process.env.FAKE_CLI_MODEL, messages: [{ role: 'user', content: prompt }] }), +}).catch((error) => ({ ok: false, status: 0, text: async () => String(error?.message || error) })); + +// double-call: a second request per turn, the way a CLI that also titles its +// session behaves. retry-5xx-forever: never gives up on a server error. +if (fault === 'double-call') await (await ask()).text(); +let response = await ask(); +while (fault === 'retry-5xx-forever' && response.status >= 500) { + await new Promise((resolve) => setTimeout(resolve, 200)); + response = await ask(); +} + +if (!response.ok) { + const body = await response.text(); + const error = fault === 'error-echo-key' ? `provider rejected key ${key}: ${body}` : body; + write({ type: 'result', ok: false, status: response.status, error }); + process.exit(1); +} + +const answer = (await response.json())?.choices?.[0]?.message?.content ?? ''; +let text = answer; +if (fault === 'echo-key') text = `${answer} (key ${key})`; +if (fault === 'extra-text') text = `${answer}\n\nLet me know if you need anything else.`; +write({ type: 'message', text }); +write({ type: 'result', ok: true, text, session_id: sessionId }); diff --git a/packages/seats/test-support/headless-fake-adapter.js b/packages/seats/test-support/headless-fake-adapter.js new file mode 100644 index 0000000000..7b90898006 --- /dev/null +++ b/packages/seats/test-support/headless-fake-adapter.js @@ -0,0 +1,48 @@ +/** + * test-support/fake-cli.mjs driven through src/adapters/headless.js, so the + * shared one-turn runner is proven against the full compat suite with a real + * child process before any third-party CLI leans on it. Also an adapter for + * a CLI that cannot reach the provider, built the way gemini, cursor and amp + * are, with fake-cli as its pinned binary. + */ +import { FAKE_CLI_PATH } from './fake-cli-adapter.js'; +import { createHeadlessAdapter, createProviderUnsupportedAdapter } from '../src/adapters/headless.js'; + +const versionArgs = [FAKE_CLI_PATH, '--version']; + +export const readFakeCliLine = (line, state) => { + const event = JSON.parse(line); + if (event.type === 'message' && event.text) state.chunks.push(event.text); + if (event.type !== 'result') return; + if (event.ok) { + state.text = event.text; + state.sessionId = event.session_id; + } else { + state.error = `${event.status}: ${event.error}`; + } +}; + +export const createHeadlessFakeAdapter = createHeadlessAdapter({ + name: 'headless-fake', + cli: 'fake-cli', + command: process.execPath, + versionArgs, + versionPattern: /fake-cli (\S+)/, + async launch(prompt, ctx, { provider, env }) { + return { + args: [FAKE_CLI_PATH, 'run', ...(ctx.sessionId ? ['--resume', ctx.sessionId] : [])], + stdin: prompt, + env: { ...env, FAKE_CLI_BASE_URL: provider.baseURL, FAKE_CLI_MODEL: provider.model, FAKE_CLI_API_KEY: provider.apiKey }, + }; + }, + read: readFakeCliLine, +}); + +export const createUnsupportedFakeAdapter = createProviderUnsupportedAdapter({ + name: 'unsupported-fake', + cli: 'fake-cli', + command: process.execPath, + versionArgs, + versionPattern: /fake-cli (\S+)/, + detail: 'this CLI only talks to its vendor\'s own service', +}); diff --git a/packages/seats/test/compat-install.test.js b/packages/seats/test/compat-install.test.js new file mode 100644 index 0000000000..5fddea9453 --- /dev/null +++ b/packages/seats/test/compat-install.test.js @@ -0,0 +1,96 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { installTarget } from '../compat/install.js'; +import { resolveBinary } from '../compat/targets.js'; + +const withTmp = async (fn) => { + const dir = mkdtempSync(join(tmpdir(), 'seats-compat-install-')); + try { return await fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); } +}; + +// A release tarball holding one executable, served from memory. +const tarball = (dir) => { + const src = join(dir, 'src'); + mkdirSync(src); + writeFileSync(join(src, 'tool'), '#!/bin/sh\necho tool 2.0.0\n'); + const archive = join(dir, 'tool.tar.gz'); + assert.equal(spawnSync('tar', ['-czf', archive, '-C', src, 'tool']).status, 0); + const body = readFileSync(archive); + return { body, sha256: createHash('sha256').update(body).digest('hex') }; +}; +const serve = (body) => async () => ({ ok: true, status: 200, arrayBuffer: async () => body }); +const target = (asset) => ({ + cli: 'tool', version: '2.0.0', adapter: 'src/adapters/stub.js#default', enabled: true, + binary: { bin: 'tool', release: { 'linux-x64': { url: 'https://example.invalid/tool.tar.gz', ...asset } } }, +}); + +test('a release asset is installed into /- only when its sha256 matches', () => withTmp(async (dir) => { + const { body, sha256 } = tarball(dir); + const prefix = join(dir, 'prefix'); + const ok = await installTarget(target({ sha256 }), { prefix, platform: 'linux-x64', fetchImpl: serve(body) }); + assert.equal(ok.installed, join(prefix, 'tool-2.0.0', 'tool')); + assert.ok(existsSync(ok.installed)); + + const other = join(dir, 'other'); + await assert.rejects( + installTarget(target({ sha256: 'a'.repeat(64) }), { prefix: other, platform: 'linux-x64', fetchImpl: serve(body) }), + /sha256 mismatch/, + ); + assert.equal(existsSync(join(other, 'tool-2.0.0', 'tool')), false, 'nothing is unpacked on a mismatch'); + await assert.rejects( + installTarget(target({}), { prefix: other, platform: 'linux-x64', fetchImpl: serve(body) }), + /no pinned sha256/, + ); +})); + +test('install skips what the runner would skip', () => withTmp(async (dir) => { + const absent = { ...target({ sha256: 'a'.repeat(64) }), adapter: 'src/adapters/not-landed.js#x' }; + assert.match((await installTarget(absent, { prefix: dir, platform: 'linux-x64' })).skipped, /not present/); + assert.match((await installTarget(target({ sha256: 'a'.repeat(64) }), { prefix: dir, platform: 'win32-x64' })).skipped, /win32-x64/); + assert.equal((await installTarget({ ...target({}), enabled: false }, { prefix: dir })).skipped, 'disabled'); +})); + +test('binary paths resolve per platform and never put the host home in the label', () => { + const npm = { cli: 'oc', version: '1.0.0', binary: { npm: 'oc@1.0.0', bin: { 'linux-x64': 'node_modules/oc-linux-x64/bin/oc' } } }; + const resolved = resolveBinary(npm, { prefix: '/opt/compat', platform: 'linux-x64' }); + assert.equal(resolved.path, '/opt/compat/oc-1.0.0/node_modules/oc-linux-x64/bin/oc'); + assert.equal(resolved.label, '/oc-1.0.0/node_modules/oc-linux-x64/bin/oc'); + assert.match(resolveBinary(npm, { prefix: '/opt', platform: 'darwin-x64' }).missing, /darwin-x64/); + assert.match(resolveBinary(npm, { platform: 'linux-x64' }).missing, /prefix/); +}); + +test('a pip spec is pinned exactly and installed into /venv from wheels only', () => withTmp(async (dir) => { + const calls = []; + const runImpl = (command, args) => { + calls.push([command, ...args]); + // Stand in for the venv and the console script pip would create. + if (args[0] === 'install') { + mkdirSync(join(dir, 'prefix', 'tool-2.0.0', 'venv', 'bin'), { recursive: true }); + writeFileSync(join(dir, 'prefix', 'tool-2.0.0', 'venv', 'bin', 'tool'), '#!/bin/sh\n'); + } + }; + const pip = { + cli: 'tool', version: '2.0.0', adapter: 'src/adapters/stub.js#default', enabled: true, + binary: { pip: 'tool-cli==2.0.0', bin: 'venv/bin/tool' }, + }; + const prefix = join(dir, 'prefix'); + const ok = await installTarget(pip, { prefix, platform: 'linux-x64', pipOptions: { python: 'python3.12', runImpl } }); + assert.equal(ok.installed, join(prefix, 'tool-2.0.0', 'venv', 'bin', 'tool')); + const venv = join(prefix, 'tool-2.0.0', 'venv'); + assert.deepEqual(calls[0], ['python3.12', '-m', 'venv', venv]); + assert.equal(calls[1][0], join(venv, 'bin', 'pip')); + assert.ok(calls[1].includes('--only-binary=:all:'), 'no sdist, so no setup.py runs'); + assert.equal(calls[1].at(-1), 'tool-cli==2.0.0'); + + for (const loose of ['tool-cli', 'tool-cli>=2', 'tool-cli==2.0.0; rm -rf /', '-r requirements.txt']) { + await assert.rejects( + installTarget({ ...pip, binary: { ...pip.binary, pip: loose } }, { prefix: join(dir, 'p2'), platform: 'linux-x64', pipOptions: { runImpl } }), + /must pin one exact version/, + ); + } +})); diff --git a/packages/seats/test/compat-probe.test.js b/packages/seats/test/compat-probe.test.js new file mode 100644 index 0000000000..6b31f6e8b0 --- /dev/null +++ b/packages/seats/test/compat-probe.test.js @@ -0,0 +1,157 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { CHECKS, readRetryProbe, runCheck } from '../compat/checks.js'; +import { tapSpawn } from '../compat/harness.js'; + +const KEY = ['sk', 'compat', 'probe', 'key', '0001'].join('-'); +const check = (id) => CHECKS.find((c) => c.id === id); +const TIMEOUTS = { turnMs: 2000, stallMs: 500, graceMs: 200, checkMs: 3000, retryWindowMs: 500 }; + +// `requests` is the harness stamp: provider requests made by that instant. +const update = (requests, sessionUpdate, extra = {}) => ({ + dir: 'in', ms: requests * 100, requests, msg: { jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'ses_1', update: { sessionUpdate, ...extra } } }, +}); +const prompt = { dir: 'out', ms: 10, requests: 0, msg: { jsonrpc: '2.0', id: 3, method: 'session/prompt', params: { sessionId: 'ses_1', prompt: [] } } }; +const retryMeta = { 'opencode/retry': { attempt: 1, nextRetryAt: 1761390000000, error: { type: 'api', message: 'fake provider failure', status: 503 } } }; +const fails = (n) => Array.from({ length: n }, () => ({ directive: 'fail' })); + +// 1.19.4 as measured in a container run: available_commands_update can land +// just after session/prompt is written, but before any provider request. +const silentTurn = () => ({ + ok: false, ms: 500, tapped: true, requests: fails(4), + error: Object.assign(new Error('timeout'), { name: 'RunFailure', kind: 'timeout' }), + events: [{ type: 'session.opened', requests: 0 }, { type: 'turn.prompt_sent', requests: 0 }, { type: 'process.exited', requests: 4 }], + wire: [prompt, update(0, 'available_commands_update')], +}); +// The shape PR #50752 describes: a session_info_update carrying only _meta. +const reportingTurn = ({ surfaced = [] } = {}) => ({ + ...silentTurn(), + events: [{ type: 'session.opened', requests: 0 }, { type: 'turn.prompt_sent', requests: 0 }, + ...surfaced.map(([type, requests]) => ({ type, requests })), { type: 'process.exited', requests: 4 }], + wire: [prompt, update(0, 'available_commands_update'), + update(2, 'session_info_update', { _meta: retryMeta }), + update(3, 'session_info_update', { _meta: { 'opencode/retry': null } }), + // After the last request: what an agent says once it has given up. + update(4, 'agent_message_chunk', { content: { type: 'text', text: 'Ran into this error' } }), + { dir: 'in', ms: 3000, requests: 4, msg: { jsonrpc: '2.0', id: 3, result: { stopReason: 'cancelled', _meta: retryMeta } } }], +}); + +const envWith = (turn) => ({ + timeouts: TIMEOUTS, + provider: { apiKey: KEY }, + detect: async () => ({ found: { path: '/bin/x', version: '1' } }), + probeTurn: turn === undefined ? undefined : () => Promise.resolve(turn), +}); + +test('readRetryProbe counts only updates between the first and the last provider request', () => { + const silent = readRetryProbe(silentTurn()); + assert.equal(silent.updates, 0, 'an update before any provider request is not retry progress'); + assert.deepEqual(silent.otherKinds, { available_commands_update: 1 }); + assert.equal(silent.failedRequests, 4); + const reporting = readRetryProbe(reportingTurn(), { key: KEY }); + assert.equal(reporting.updates, 2); + assert.deepEqual(reporting.kinds, { session_info_update: 2 }); + assert.deepEqual(reporting.otherKinds, { available_commands_update: 1, agent_message_chunk: 1 }, 'the give-up text is not progress'); + assert.deepEqual(reporting.metaKeys, ['opencode/retry']); + assert.deepEqual(reporting.answerMetaKeys, ['opencode/retry']); + assert.deepEqual(reporting.surfaced, []); + assert.match(reporting.sample, /"sessionUpdate":"session_info_update"/); +}); + +test('readRetryProbe scrubs the key from the sample it quotes', () => { + const turn = reportingTurn(); + turn.wire[2] = update(2, 'session_info_update', { _meta: { echo: KEY } }); + const probe = readRetryProbe(turn, { key: KEY }); + assert.doesNotMatch(probe.sample, new RegExp(KEY)); + assert.match(probe.sample, //); +}); + +test('retry-status fails a silent retry and passes one the agent reports', async () => { + const silent = await runCheck(check('retry-status'), {}, envWith(silentTurn())); + assert.equal(silent.status, 'fail'); + assert.match(silent.detail, /4 provider requests in 500ms and no session\/update between the first and the last; outside the retries: available_commands_update×1/); + const reporting = await runCheck(check('retry-status'), {}, envWith(reportingTurn())); + assert.equal(reporting.status, 'pass', reporting.detail); + assert.match(reporting.detail, /2 session\/update\(s\) between the first and the last of 4 provider requests in 500ms: session_info_update×2; _meta keys opencode\/retry; outside the retries: available_commands_update×1, agent_message_chunk×1; the prompt answer carries _meta opencode\/retry/); +}); + +test('retry-status passes an agent that does not retry, and skips when it cannot see the agent', async () => { + const once = { ...silentTurn(), requests: fails(1) }; + assert.equal((await runCheck(check('retry-status'), {}, envWith(once))).status, 'pass'); + const noProbe = await runCheck(check('retry-status'), {}, envWith(undefined)); + assert.equal(noProbe.status, 'skip'); + assert.match(noProbe.detail, /declares no probeOptions/); + const untapped = await runCheck(check('retry-status'), {}, envWith({ ...silentTurn(), tapped: false, wire: [] })); + assert.equal(untapped.status, 'skip'); + assert.match(untapped.detail, /took no spawn hook/); + const noPrompt = await runCheck(check('retry-status'), {}, envWith({ ...silentTurn(), wire: [] })); + assert.equal(noPrompt.status, 'skip'); +}); + +test('retry-status-surfaced fails when the trace drops what the agent reported', async () => { + const dropped = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn())); + assert.equal(dropped.status, 'fail'); + assert.match(dropped.detail, /the agent sent session_info_update×2 \(_meta keys opencode\/retry\); the trace carries no event from them/); + const carried = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn({ surfaced: [['agent.thought_chunk', 2], ['agent.thought_chunk', 3]] }))); + assert.equal(carried.status, 'pass', carried.detail); + // An event from the give-up text does not stand in for a dropped retry report. + const late = await runCheck(check('retry-status-surfaced'), {}, envWith(reportingTurn({ surfaced: [['agent.message_chunk', 4], ['usage', 4]] }))); + assert.equal(late.status, 'fail', late.detail); + const nothing = await runCheck(check('retry-status-surfaced'), {}, envWith(silentTurn())); + assert.equal(nothing.status, 'skip'); +}); + +test('tapSpawn records JSON lines both ways, split across chunks, and drops the rest', async () => { + const wire = []; + const script = [ + "process.stdin.on('data', (d) => {", + " process.stdout.write('{\"jsonrpc\":\"2.0\",\"id\":1,');", + " setTimeout(() => { process.stdout.write('\"result\":{}}\\nnot json\\n'); process.exit(0); }, 20);", + '});', + ].join('\n'); + let n = 0; + const child = tapSpawn(wire, undefined, { stamp: () => ({ requests: n++ }) })(process.execPath, ['-e', script], { stdio: ['pipe', 'pipe', 'pipe'] }); + child.stdout.on('data', () => {}); // the adapter's own reader + child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize' })}\n`); + await new Promise((resolve) => child.once('close', resolve)); + assert.deepEqual(wire.map((w) => [w.dir, w.msg.method ?? 'answer']), [['out', 'initialize'], ['in', 'answer']]); + assert.ok(wire.every((w) => Number.isFinite(w.ms))); + assert.deepEqual(wire.map((w) => w.requests), [0, 1], 'each record carries the stamp'); +}); + +// goose 1.56.0 as measured: a run marker before any request, two session-state +// updates beside the first, then nothing until its error text after the last. +test('start-of-turn bookkeeping and session state are not retry progress', async () => { + const turn = { + ...silentTurn(), + requests: fails(8), + events: [{ type: 'turn.prompt_sent', requests: 0 }, { type: 'usage', requests: 1 }, { type: 'usage', requests: 3 }, { type: 'agent.message_chunk', requests: 8 }], + wire: [prompt, + update(0, 'session_info_update', { _meta: { goose: { activeRunId: 'run_1' } } }), + update(1, 'usage_update', { used: 0, size: 128000 }), + update(1, 'available_commands_update', { availableCommands: [] }), + update(3, 'usage_update', { used: 0, size: 128000 }), + update(8, 'agent_message_chunk', { content: { type: 'text', text: 'Ran into this error: 503' } }), + update(8, 'session_info_update', { _meta: { goose: { activeRunId: null } } })], + }; + const probe = readRetryProbe(turn); + assert.equal(probe.updates, 0, JSON.stringify(probe)); + assert.deepEqual(probe.surfaced, [], 'a usage event is session state, not a surfaced retry report'); + const result = await runCheck(check('retry-status'), {}, envWith(turn)); + assert.equal(result.status, 'fail'); + assert.match(result.detail, /8 provider requests .* no session\/update between the first and the last; outside the retries: session_info_update×2, usage_update×2, available_commands_update×1, agent_message_chunk×1/); +}); + +// While only the first request is out nothing has been retried yet, so what +// the agent streams then (a thought, a run marker) is not a retry report. +test('an update while only the first request is out is not retry progress', () => { + const turn = { + ...silentTurn(), + wire: [prompt, + update(1, 'agent_thought_chunk', { content: { type: 'text', text: 'thinking' } }), + update(1, 'session_info_update', { _meta: { goose: { activeRunId: 'run_1' } } })], + }; + const probe = readRetryProbe(turn); + assert.equal(probe.updates, 0, JSON.stringify(probe)); + assert.deepEqual(probe.otherKinds, { agent_thought_chunk: 1, session_info_update: 1 }); +}); diff --git a/packages/seats/test/compat-render.test.js b/packages/seats/test/compat-render.test.js new file mode 100644 index 0000000000..2ecbfc8b8b --- /dev/null +++ b/packages/seats/test/compat-render.test.js @@ -0,0 +1,83 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { escapeHtml, limitText, renderMatrix } from '../compat/render.js'; + +const results = (detail) => ({ + schema: 'seats-compat/1', + generatedAt: '2026-10-25T00:00:00.000Z', + host: { os: 'linux 6.1', platform: 'linux', arch: 'x64', node: 'v22.0.0' }, + checks: [{ id: 'detect', title: 'detect() finds binary' }, { id: 'refusal', title: 'refusal' }], + targets: [{ target: 'evil@1">', cli: 'evil', version: '1', note: 'note' }], + rows: [ + { target: 'evil@1">', check: 'detect', status: 'pass', detail: 'ok', ms: 3 }, + { target: 'evil@1">', check: 'refusal', status: 'fail', detail, ms: 9 }, + ], +}); + +test('the renderer escapes a & more')); + assert.doesNotMatch(html, /