diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index a547e07ba1..78e1641fb1 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -49,6 +49,7 @@ on: branches: [ main ] paths: - 'cli/**' + - 'packages/seats/**' # bundled into the cli tarball (bundleDependencies) - 'commonly-mcp/**' - 'docs/agents/skills/commonly/SKILL.md' # cli prepublishOnly copies it in workflow_dispatch: @@ -112,6 +113,14 @@ jobs: with: node-version: '22' + # Before the compare, not just before publish: the cli bundles + # packages/seats, and `npm pack` on a tree with no node_modules exits 0 + # with the bundle silently missing. The compare would then see every + # tree as different from what npm serves. cli's prepack now refuses + # that case, so a missing install fails here, by name. + - name: Install + run: npm ci --ignore-scripts + - name: Decide whether this tree holds anything unpublished id: cmp shell: bash @@ -237,10 +246,6 @@ jobs: done true - - name: Install - if: steps.cmp.outputs.action == 'publish' - run: npm ci --ignore-scripts - - name: Publish if: steps.cmp.outputs.action == 'publish' # No NODE_AUTH_TOKEN: npm exchanges this job's OIDC id-token with the diff --git a/.gitleaks.toml b/.gitleaks.toml index 60cabd2423..95360d5fcb 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -22,6 +22,11 @@ regexes = [ # signature. generic-api-key matched it, and main's Secret Scan was red on # every push from 2026-10-16. The exact literal only, so a real token still fails. '''eyJhbGciOiJSUzI1NiJ9\.jwt''', + # A fake OpenRouter key in packages/seats/test/supervisor.test.js (#2701): the + # scrub test needs a value shaped like a real key so it can prove the key never + # reaches the record. generic-api-key matched it in a commit the scan reads, so + # an inline allow on today's line would not clear it. The exact literal only. + '''sk-or-v1-abcdefabcdefabcdef''', # The namespaced cloudflared ExternalSecret has `- secretKey: credentials.json` # (#2439) — a Secret KEY NAME (what the tunnel credentials.json is stored under), not # a value. gitleaks' kubernetes-secret-yaml rule captures `secretKey: credentials` (it diff --git a/cli/__tests__/adapters.process-group.test.mjs b/cli/__tests__/adapters.process-group.test.mjs new file mode 100644 index 0000000000..715b5becd4 --- /dev/null +++ b/cli/__tests__/adapters.process-group.test.mjs @@ -0,0 +1,67 @@ +/** + * The claude and codex adapters start their child in its own process group + * only when the run supervisor asks (ctx.ownProcessGroup), because only the + * supervisor kills the group. Live seats run unsupervised, and their spawn + * options must stay exactly what they were. + */ +import { jest } from '@jest/globals'; +import { EventEmitter } from 'events'; + +await jest.unstable_mockModule('child_process', () => ({ + spawnSync: jest.fn(() => ({ status: 1, stdout: '', stderr: '' })), + spawn: jest.fn(), +})); + +const claude = (await import('../src/lib/adapters/claude.js')).default; +const codex = (await import('../src/lib/adapters/codex.js')).default; +const { LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP } = await import('@commonlyai/seats/process-group'); + +const makeSpawnImpl = () => { + const calls = []; + const impl = (cmd, args, opts) => { + const proc = new EventEmitter(); + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.kill = jest.fn(); + calls.push({ cmd, args, opts, proc }); + setTimeout(() => { + proc.stdout.emit('data', Buffer.from('ok')); + proc.emit('close', 0); + }, 0); + return proc; + }; + return { impl, calls }; +}; + +const ADAPTERS = [['claude', claude], ['codex', codex]]; +const POSIX = process.platform !== 'win32'; + +describe.each(ADAPTERS)('%s adapter — process group', (name, adapter) => { + test('unsupervised: spawns with exactly cwd, env and stdio, no detached, no marker', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl }); + expect(calls).toHaveLength(1); + expect(Object.keys(calls[0].opts).sort()).toEqual(['cwd', 'env', 'stdio']); + expect(calls[0].opts.stdio).toEqual(['ignore', 'pipe', 'pipe']); + expect(calls[0].proc[LEADS_PROCESS_GROUP]).toBeUndefined(); + }); + + (POSIX ? test : test.skip)('supervised: spawns detached and marks the child as leading its group', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { + environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl, [OWN_PROCESS_GROUP]: true, + }); + expect(calls).toHaveLength(1); + expect(calls[0].opts.detached).toBe(true); + expect(Object.keys(calls[0].opts).sort()).toEqual(['cwd', 'detached', 'env', 'stdio']); + expect(calls[0].proc[LEADS_PROCESS_GROUP]).toBe(true); + }); + + test('a truthy non-true value does not opt in', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { + environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl, [OWN_PROCESS_GROUP]: 'yes', + }); + expect(calls[0].opts).not.toHaveProperty('detached'); + }); +}); diff --git a/cli/__tests__/seat-stop.process.test.mjs b/cli/__tests__/seat-stop.process.test.mjs new file mode 100644 index 0000000000..d4433402f4 --- /dev/null +++ b/cli/__tests__/seat-stop.process.test.mjs @@ -0,0 +1,136 @@ +/** + * seat-stop with real processes, wired the way `agent run` wires it: SIGTERM + * calls requestStop, and the exit path calls onExit. The supervised child + * leads its own group and holds a grandchild (`sleep 300 &`), the shape of a + * CLI holding an MCP server. + * + * - A launchd seat (its plist exists): the groups die a margin before + * ExitTimeOut, while the turn is still running. + * - A daemon-spawned seat (no plist): nothing is armed, the turn runs past the + * 15 s a default deadline would have cut it at, and the groups die on exit. + */ +import { spawn } from 'child_process'; +import { once } from 'events'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SEAT_STOP = pathToFileURL(join(HERE, '..', 'src', 'lib', 'seat-stop.js')).href; +const SEATS = join(HERE, '..', '..', 'packages', 'seats', 'src'); +const seats = (file) => JSON.stringify(pathToFileURL(join(SEATS, file)).href); +const POSIX = process.platform !== 'win32'; + +const isRunning = (pid) => { + try { process.kill(pid, 0); } catch (error) { return error.code !== 'ESRCH'; } + try { return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return true; } +}; +const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); }); +const waitFor = async (predicate, ms = 5000) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (predicate()) return true; + await sleep(25); + } + return predicate(); +}; +const reap = (pid) => { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }; + +const HOST = ({ home, pidFile, goFile }) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { createSeatStop } from ${JSON.stringify(SEAT_STOP)}; +import { superviseRun } from ${seats('supervisor.js')}; +import { createTraceStore } from ${seats('trace/store.js')}; +import { observeSpawn } from ${seats('runtime-observation.js')}; +import { spawnInOwnGroup, wantsOwnProcessGroup } from ${seats('process-group.js')}; +let stopping = false; +const seatStop = createSeatStop({ name: 'probe', stop: () => { stopping = true; }, home: ${JSON.stringify(home)} }); +process.on('SIGTERM', (signal) => { process.stdout.write('stopping deadline=' + seatStop.requestStop(signal) + '\\n'); }); +process.once('exit', seatStop.onExit); +const script = "sleep 300 & echo $! > '${pidFile}'; while [ ! -f '${goFile}' ]; do sleep 0.05; done; echo done; wait"; +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', script], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.stdout.on('data', (chunk) => { if (String(chunk).includes('done')) resolve({ text: 'done' }); }); + child.on('close', (code, signal) => reject(new Error('sh ended ' + (code ?? signal)))); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + }, 20); + }); +}) }; +const out = await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(join(home, 'trace'))} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +process.stdout.write('turn ' + out.outcome + '\\n'); +if (stopping) process.exit(0); +`; + +let home; +let state; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'seat-stop-proc-')); + state = null; +}); +afterEach(() => { + if (state) { + if (state.grandchild) reap(state.grandchild); + if (state.leader) reap(state.leader); + state.host.kill('SIGKILL'); + } + rmSync(home, { recursive: true, force: true }); +}); + +const startHost = async () => { + const pidFile = join(home, 'grandchild.pid'); + const goFile = join(home, 'go'); + const script = join(home, 'host.mjs'); + writeFileSync(script, HOST({ home, pidFile, goFile })); + const host = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + state = { host, goFile, out: '' }; + host.stdout.on('data', (chunk) => { state.out += chunk; }); + state.exited = once(host, 'exit'); + expect(await waitFor(() => /ready \d+/.test(state.out))).toBe(true); + state.leader = Number(state.out.match(/ready (\d+)/)[1]); + expect(await waitFor(() => existsSync(pidFile) && /^\d+\s*$/.test(readFileSync(pidFile, 'utf8')))).toBe(true); + state.grandchild = Number(readFileSync(pidFile, 'utf8')); + return state; +}; + +const describePosix = POSIX ? describe : describe.skip; + +describePosix('seat-stop — real processes', () => { + test('a seat with a plist: the groups die a margin before ExitTimeOut, mid-turn', async () => { + // ExitTimeOut 2 s → deadline 1.5 s. + mkdirSync(join(home, 'Library', 'LaunchAgents'), { recursive: true }); + writeFileSync(join(home, 'Library', 'LaunchAgents', 'me.commonly.seat-probe.plist'), + 'ExitTimeOut2'); + await startHost(); + const sent = Date.now(); + state.host.kill('SIGTERM'); + expect(await waitFor(() => /stopping deadline=1500/.test(state.out))).toBe(true); + await sleep(750); + expect(isRunning(state.grandchild)).toBe(true); + expect(await waitFor(() => !isRunning(state.grandchild), 2000)).toBe(true); + expect(Date.now() - sent).toBeLessThan(2000); + expect(existsSync(state.goFile)).toBe(false); + }, 30000); + + test('a seat with no plist: nothing is armed, the turn runs past 15 s, and the groups die on exit', async () => { + await startHost(); + state.host.kill('SIGTERM'); + expect(await waitFor(() => /stopping deadline=null/.test(state.out))).toBe(true); + await sleep(16000); + expect(isRunning(state.grandchild)).toBe(true); + expect(isRunning(state.leader)).toBe(true); + writeFileSync(state.goFile, ''); + const [code] = await Promise.race([state.exited, sleep(5000).then(() => ['timeout'])]); + expect(state.out).toMatch(/turn ok/); + expect(code).toBe(0); + expect(await waitFor(() => !isRunning(state.grandchild))).toBe(true); + expect(await waitFor(() => !isRunning(state.leader))).toBe(true); + }, 40000); +}); diff --git a/cli/__tests__/seat-stop.test.mjs b/cli/__tests__/seat-stop.test.mjs new file mode 100644 index 0000000000..42934793ae --- /dev/null +++ b/cli/__tests__/seat-stop.test.mjs @@ -0,0 +1,128 @@ +/** + * seat-stop: SIGTERM keeps the turn, and arms a supervised-group kill a margin + * before the seat plist's ExitTimeOut (launchd's SIGKILL). The plist is only + * read. The process-level behaviour is covered with real processes in + * packages/seats/test/supervisor-tree-and-deadline.test.js. + */ +import { jest } from '@jest/globals'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { + DEFAULT_EXIT_TIMEOUT_S, createSeatStop, groupKillDeadlineMs, readExitTimeOut, readSeatPlist, seatPlistPath, +} from '../src/lib/seat-stop.js'; + +const plist = (body) => ` + + + + Label + me.commonly.seat-anvil + KeepAlive + +${body} + + +`; + +let home; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'seat-stop-')); + mkdirSync(join(home, 'Library', 'LaunchAgents'), { recursive: true }); +}); +afterEach(() => { rmSync(home, { recursive: true, force: true }); }); + +const writePlist = (name, body) => { + const path = seatPlistPath(name, home); + writeFileSync(path, plist(body)); + return path; +}; + +describe('seat-stop — ExitTimeOut', () => { + test('the plist path is the seat launch agent', () => { + expect(seatPlistPath('anvil', home)).toBe(join(home, 'Library', 'LaunchAgents', 'me.commonly.seat-anvil.plist')); + expect(seatPlistPath('../x', home)).toBeNull(); + expect(seatPlistPath('a/b', home)).toBeNull(); + }); + + test('reads ExitTimeOut from an XML plist, and only reads it', () => { + const path = writePlist('anvil', ' ExitTimeOut\n 40'); + const before = readFileSync(path, 'utf8'); + expect(readExitTimeOut(path)).toBe(40); + expect(readFileSync(path, 'utf8')).toBe(before); + }); + + test('unset, zero, missing file and no path all read as unset', () => { + expect(readExitTimeOut(writePlist('a', ''))).toBeNull(); + expect(readExitTimeOut(writePlist('b', ' ExitTimeOut\n 0'))).toBeNull(); + expect(readExitTimeOut(join(home, 'nope.plist'))).toBeNull(); + expect(readExitTimeOut(null)).toBeNull(); + }); + + test('a binary plist is read through plutil -extract', () => { + const run = jest.fn(() => ({ status: 0, stdout: '30\n' })); + const value = readExitTimeOut('/x.plist', { readFile: () => Buffer.from('bplist00...'), run }); + expect(value).toBe(30); + expect(run).toHaveBeenCalledWith('plutil', ['-extract', 'ExitTimeOut', 'raw', '-o', '-', '/x.plist'], expect.any(Object)); + }); + + test('the deadline is ExitTimeOut less min(5 s, a quarter); unset means 20 s', () => { + expect(DEFAULT_EXIT_TIMEOUT_S).toBe(20); + expect(groupKillDeadlineMs(null)).toBe(15000); + expect(groupKillDeadlineMs(40)).toBe(35000); + expect(groupKillDeadlineMs(4)).toBe(3000); + for (const s of [1, 4, 20, 40, 120]) expect(groupKillDeadlineMs(s)).toBeLessThan(s * 1000); + }); +}); + +describe('seat-stop — requestStop', () => { + const make = (name = 'anvil') => { + const stop = jest.fn(); + const arm = jest.fn(); + const kill = jest.fn(); + return { stop, arm, kill, seatStop: createSeatStop({ name, stop, home, deps: { arm, kill } }) }; + }; + + test('SIGTERM stops the loop and arms the group kill before ExitTimeOut', () => { + writePlist('anvil', ' ExitTimeOut\n 8'); + const { stop, arm, kill, seatStop } = make(); + expect(seatStop.requestStop('SIGTERM')).toBe(6000); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).toHaveBeenCalledWith(6000); + expect(kill).not.toHaveBeenCalled(); + }); + + test('SIGTERM on a seat with no plist (daemon-spawned) arms nothing: the exit path alone kills', () => { + const { stop, arm, seatStop } = make('daemon-seat'); + expect(seatStop.requestStop('SIGTERM')).toBeNull(); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).not.toHaveBeenCalled(); + }); + + test('SIGTERM on a seat whose plist has no ExitTimeOut uses launchd\'s 20 s default', () => { + writePlist('anvil', ''); + const { arm, seatStop } = make(); + expect(seatStop.requestStop('SIGTERM')).toBe(15000); + expect(arm).toHaveBeenCalledWith(15000); + }); + + test('a plist that exists but cannot be read still counts as present, at the default', () => { + expect(readSeatPlist('/p.plist', { exists: () => true, readFile: () => { throw new Error('EACCES'); } })) + .toEqual({ present: true, exitTimeOut: null }); + expect(readSeatPlist(join(home, 'nope.plist'))).toEqual({ present: false, exitTimeOut: null }); + expect(readSeatPlist(null)).toEqual({ present: false, exitTimeOut: null }); + }); + + test('SIGINT stops the loop and arms nothing', () => { + const { stop, arm, seatStop } = make(); + expect(seatStop.requestStop('SIGINT')).toBeNull(); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).not.toHaveBeenCalled(); + }); + + test('the exit path kills the supervised groups', () => { + const { kill, seatStop } = make(); + seatStop.onExit(); + expect(kill).toHaveBeenCalledTimes(1); + }); +}); diff --git a/cli/__tests__/seats-package.test.mjs b/cli/__tests__/seats-package.test.mjs new file mode 100644 index 0000000000..ef8bb984f8 --- /dev/null +++ b/cli/__tests__/seats-package.test.mjs @@ -0,0 +1,80 @@ +/** + * packages/seats is the runtime the CLI bundles (D62, extract-in-place). These + * tests hold the two properties the move depends on: the package imports + * nothing from Commonly, and the CLI tarball carries it. + */ +import { execFileSync } from 'child_process'; +import { readdirSync, readFileSync, statSync } from 'fs'; +import { builtinModules } from 'module'; +import { dirname, join, relative, resolve, sep } from 'path'; +import { fileURLToPath } from 'url'; +import { createHermesAdapter, createRegistry, stub } from '@commonlyai/seats'; +import cliHermes from '../src/lib/adapters/hermes.js'; +import { getAdapter } from '../src/lib/adapters/index.js'; + +const CLI = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const SEATS = resolve(CLI, '..', 'packages', 'seats'); + +const sourceFiles = (dir) => readdirSync(dir).flatMap((name) => { + const full = join(dir, name); + return statSync(full).isDirectory() ? sourceFiles(full) : (full.endsWith('.js') ? [full] : []); +}); + +const specifiers = (code) => [ + ...code.matchAll(/(?:^|\n)\s*(?:import|export)\b[^'"`;]*?from\s*['"]([^'"]+)['"]/g), + ...code.matchAll(/(?:^|\n)\s*import\s*['"]([^'"]+)['"]/g), + ...code.matchAll(/\bimport\(\s*['"]([^'"]+)['"]\s*\)/g), +].map((match) => match[1]); + +describe('packages/seats', () => { + test('imports only node built-ins and its own modules', () => { + const builtins = new Set(builtinModules); + const files = sourceFiles(join(SEATS, 'src')); + expect(files.length).toBeGreaterThan(5); + const outside = []; + for (const file of files) { + for (const spec of specifiers(readFileSync(file, 'utf8'))) { + if (spec.startsWith('node:') || builtins.has(spec)) continue; + const target = spec.startsWith('.') ? resolve(dirname(file), spec) : null; + if (target && (target + sep).startsWith(join(SEATS, 'src') + sep)) continue; + outside.push(`${relative(SEATS, file)} → ${spec}`); + } + } + expect(outside).toEqual([]); + }); + + test('the boundary scan sees a bad import', () => { + expect(specifiers("import x from '../../cli/src/lib/environment.js';\nexport { y } from 'commander';")) + .toEqual(['../../cli/src/lib/environment.js', 'commander']); + }); + + test('the CLI registry serves the bound adapters', () => { + expect(getAdapter('stub')).toBe(stub); + expect(getAdapter('hermes')).toBe(cliHermes); + expect(Object.keys(cliHermes).sort()).toEqual( + Object.keys(createHermesAdapter()).sort(), + ); + }); + + test('a host-less Hermes adapter takes the neutral defaults', async () => { + const hermes = createHermesAdapter(); + expect(hermes.requiresModel).toBe(false); + await expect(hermes.validateEnvironment({ sandbox: { trust: 'public' } })) + .rejects.toThrow('no enforced confinement adapter exists'); + }); + + test('a registry refuses duplicate names', () => { + expect(() => createRegistry([stub, stub])).toThrow('duplicate adapter name: stub'); + expect(createRegistry([stub]).get('stub')).toBe(stub); + }); + + test('the CLI tarball bundles every package source file', () => { + const [pack] = JSON.parse(execFileSync('npm', ['pack', '--dry-run', '--json', '--ignore-scripts'], { + cwd: CLI, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], + })); + const packed = new Set(pack.files.map((file) => file.path)); + const expected = sourceFiles(join(SEATS, 'src')) + .map((file) => `node_modules/@commonlyai/seats/${relative(SEATS, file).split(sep).join('/')}`); + expect(expected.filter((path) => !packed.has(path))).toEqual([]); + }, 60000); +}); diff --git a/cli/package-lock.json b/cli/package-lock.json index a987ab49eb..bf644e7f0c 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -7,8 +7,12 @@ "": { "name": "@commonlyai/cli", "version": "0.1.168", + "bundleDependencies": [ + "@commonlyai/seats" + ], "license": "Apache-2.0", "dependencies": { + "@commonlyai/seats": "file:../packages/seats", "commander": "^12.0.0" }, "bin": { @@ -22,6 +26,14 @@ "node": ">=20.0.0" } }, + "../packages/seats": { + "name": "@commonlyai/seats", + "version": "0.0.0", + "license": "Apache-2.0", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -518,6 +530,10 @@ "dev": true, "license": "MIT" }, + "node_modules/@commonlyai/seats": { + "resolved": "../packages/seats", + "link": true + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", diff --git a/cli/package.json b/cli/package.json index b669f271b1..fc5a0e3441 100644 --- a/cli/package.json +++ b/cli/package.json @@ -32,11 +32,16 @@ "lint:fix": "eslint src --ext .js --fix", "test": "node --experimental-vm-modules node_modules/.bin/jest", "test:temp-cleanup": "node scripts/verify-temp-cleanup.mjs", + "prepack": "node scripts/require-bundled-seats.mjs", "prepublishOnly": "node -e \"require('fs').copyFileSync('../docs/agents/skills/commonly/SKILL.md','skills/commonly/SKILL.md')\"" }, "dependencies": { + "@commonlyai/seats": "file:../packages/seats", "commander": "^12.0.0" }, + "bundleDependencies": [ + "@commonlyai/seats" + ], "devDependencies": { "eslint": "^8.56.0", "jest": "^29.7.0" diff --git a/cli/scripts/require-bundled-seats.mjs b/cli/scripts/require-bundled-seats.mjs new file mode 100644 index 0000000000..63aaaf4f76 --- /dev/null +++ b/cli/scripts/require-bundled-seats.mjs @@ -0,0 +1,11 @@ +// prepack: refuse to pack the CLI without its bundled runtime package. +// `npm pack` on a tree where npm ci has not run exits 0 and silently leaves +// node_modules/@commonlyai/seats out of the tarball, which would publish a CLI +// whose adapters cannot load. +import { existsSync } from 'fs'; + +const manifest = new URL('../node_modules/@commonlyai/seats/package.json', import.meta.url); +if (!existsSync(manifest)) { + console.error('@commonlyai/seats is not installed in cli/node_modules; run `npm ci` in cli/ before packing or publishing'); + process.exit(1); +} diff --git a/cli/src/commands/agent.js b/cli/src/commands/agent.js index c0e5978bdf..af4691d944 100644 --- a/cli/src/commands/agent.js +++ b/cli/src/commands/agent.js @@ -82,6 +82,7 @@ import { detectBwrap } from '../lib/sandbox/bwrap.js'; import { resolvePublicSandboxMode } from '../lib/sandbox/mode.js'; import { hostSandboxRefusal, windowsSandboxMessage } from '../lib/sandbox/host-check.js'; import { prepareMcpBeforeFirstPoll } from '../lib/mcp-first-run.js'; +import { createSeatStop } from '../lib/seat-stop.js'; import { detectSeatbelt } from '../lib/sandbox/seatbelt.js'; import { describeUpstreamRefusal } from '../lib/upstream-refusal.js'; import { @@ -3836,12 +3837,16 @@ Docs: // SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops // future polls but lets the in-flight turn finish and ack before the // event loop drains; exiting here would cut a model turn in half. - const requestStop = () => { + // seat-stop also arms the supervised-group deadline before launchd's + // SIGKILL, and kills any group still live when the process exits. + const seatStop = createSeatStop({ name, stop }); + const requestStop = (signal) => { console.log(`\n${stamp()} [${name}] stopping after the current turn...`); - stop(); + seatStop.requestStop(signal); }; process.on('SIGINT', requestStop); process.on('SIGTERM', requestStop); + process.once('exit', seatStop.onExit); }); // ── detach (ADR-005) ────────────────────────────────────────────────────── diff --git a/cli/src/lib/adapters/claude.js b/cli/src/lib/adapters/claude.js index 75a17b48dd..ff96cc1dfa 100644 --- a/cli/src/lib/adapters/claude.js +++ b/cli/src/lib/adapters/claude.js @@ -1,5 +1,6 @@ import { claudeRuntimeInventory } from '../native-runtime-inventory.js'; import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group'; /** * claude adapter — wraps the local `claude` CLI as a Commonly agent. * @@ -301,12 +302,14 @@ const publicSkillPromptArgs = async (skillSources) => { return { args: text ? ['--append-system-prompt', text] : [], included }; }; -const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {} }) => new Promise((resolve, reject) => { - const proc = spawnImpl(cmd, args, { +// `ownGroup` is set only under the run supervisor, which kills the child's +// whole group; unsupervised, the spawn options are exactly what they were. +const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false }) => new Promise((resolve, reject) => { + const proc = spawnInOwnGroup(spawnImpl, cmd, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], - }); + }, { ownGroup }); observeSpawn(proc, onSpawn, controls); let stdout = ''; let stderr = ''; @@ -920,6 +923,7 @@ export default { // the value a tail can echo back. credentials: spawnCredentials(spawnCtx), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }), // test seam only — do not use in production + ownGroup: wantsOwnProcessGroup(ctx), }); // TASK-440: the context this turn left, read from the transcript the // child wrote under its own env (an isolated HOME for public seats). @@ -974,6 +978,7 @@ export default { timeoutMs: ctx.timeoutMs || DEFAULT_TIMEOUT_MS, credentials: spawnCredentials(retryCtx), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }), + ownGroup: wantsOwnProcessGroup(ctx), }); const contextTokens = claudeContextTokens({ cwd: ctx.cwd, sessionId: freshId, diff --git a/cli/src/lib/adapters/codex.js b/cli/src/lib/adapters/codex.js index 2b417acb74..4d1b6979e5 100644 --- a/cli/src/lib/adapters/codex.js +++ b/cli/src/lib/adapters/codex.js @@ -1,5 +1,6 @@ import { codexRuntimeInventory } from '../native-runtime-inventory.js'; import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group'; /** * codex adapter — wraps the local `codex` CLI as a Commonly agent. * @@ -572,14 +573,15 @@ const CODEX_PROVIDER_KEY_ENVS = [ 'CODEX_API_KEY', ]; -const runCodex = ({ args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {} }) => new Promise((resolve, reject) => { // stdio: ['ignore', 'pipe', 'pipe'] — without this, child_process.spawn +// `ownGroup`: see runClaude in claude.js. +const runCodex = ({ args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false }) => new Promise((resolve, reject) => { // stdio: ['ignore', 'pipe', 'pipe'] — without this, child_process.spawn // defaults stdin to a fresh pipe. Codex 0.125.0's `exec` then blocks on // `Reading additional input from stdin...` because it sees an open pipe // and waits for input that never arrives. Interactive runs are fine because // codex detects a TTY and uses the argv prompt directly. Setting stdin to // `'ignore'` gives codex /dev/null → immediate EOF → it falls back to the // argv prompt as intended. Surfaced live during ADR-005 Phase 2 smoke. - const proc = spawnImpl('codex', args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }); + const proc = spawnInOwnGroup(spawnImpl, 'codex', args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup }); observeSpawn(proc, onSpawn, controls); let stderr = ''; let timedOut = false; @@ -745,6 +747,7 @@ export default { // nothing left to compare a tail against. credentials: spawnCredentials(ctx, CODEX_PROVIDER_KEY_ENVS), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args.slice(args.indexOf('exec') + (args[args.indexOf('exec') + 1] === 'resume' ? 3 : 1), -1), { modelFlag: '--model', valueFlags: ['-c', '-o', '--permission-profile'], codex: true }), // test seam only — do not use in production + ownGroup: wantsOwnProcessGroup(ctx), }); let text = ''; diff --git a/cli/src/lib/adapters/hermes.js b/cli/src/lib/adapters/hermes.js index 2c6906c620..8181bbe1f2 100644 --- a/cli/src/lib/adapters/hermes.js +++ b/cli/src/lib/adapters/hermes.js @@ -1,199 +1,18 @@ -import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; -/** Hermes 0.22.2 local CLI wrapper. Hermes owns provider I/O; this file only spawns it. */ -import { requireSeatModel, seatModelRequired } from './seat-model.js'; -import { spawn as childSpawn, spawnSync } from 'child_process'; -import { createHash } from 'crypto'; -import { mkdir } from 'fs/promises'; import { homedir } from 'os'; import { join } from 'path'; -import { StringDecoder } from 'string_decoder'; +import { createHermesAdapter } from '@commonlyai/seats/adapters/hermes'; import { buildMemoryPreamble } from '../memory-bridge.js'; import { effectiveSandboxTrust } from '../environment.js'; -import { adapterFailure, spawnCredentials } from '../upstream-refusal.js'; - -export const TESTED_HERMES_VERSION = '0.22.2'; -const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; -const VERSION_TIMEOUT_MS = 15000; -const MAX_STDERR_CHARS = 4096; -const MAX_LINE_CHARS = 1024 * 1024; - -const readVersion = (env, spawnSyncImpl = spawnSync) => { - let probe; - for (let attempt = 0; attempt < 2; attempt += 1) { - probe = spawnSyncImpl('hermes', ['--version'], { - encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS, - }); - if (probe.error?.code === 'ENOENT') return null; - if (probe.error || probe.status !== 0) continue; - const version = String(probe.stdout || '').match(/Hermes Agent v([^\s]+)/i)?.[1]; - if (version !== TESTED_HERMES_VERSION) { - throw new Error(`Hermes ${version || 'unknown'} is unsupported; this adapter requires ${TESTED_HERMES_VERSION}`); - } - return version; - } - const status = probe.status ?? 'none'; - const signal = probe.signal ?? 'none'; - const code = probe.error?.code ?? 'none'; - throw new Error(`Hermes version probe failed after 2 attempts (status=${status}, signal=${signal}, code=${code}, timeout=${VERSION_TIMEOUT_MS}ms); check the Hermes installation or startup log`); -}; - -export const readHermesJsonl = (line, state) => { - let event; - try { event = JSON.parse(line); } catch { throw new Error('Hermes emitted invalid JSONL'); } - if (state.result) throw new Error('Hermes emitted data after its terminal result'); - if (event?.type === 'system' && event.subtype === 'init') { - state.started = true; - } else if (event?.type === 'result') { - state.result = event; - } else if (!state.started) { - throw new Error('Hermes emitted an event before init'); - } -}; - -const runHermes = ({ args, prompt, cwd, env, timeoutMs, spawnImpl, credentials, onSpawn, controls = {} }) => new Promise((resolve, reject) => { - const child = (spawnImpl || childSpawn)('hermes', args, { - cwd, env, stdio: ['pipe', 'pipe', 'pipe'], - }); - observeSpawn(child, onSpawn, controls); - const state = { started: false, result: null }; - const decoder = new StringDecoder('utf8'); - let buffer = ''; - let stderr = ''; - let timedOut = false; - let settled = false; - let closed = false; - let stopping = false; - let killTimer; - const finish = (error, result) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) reject(error); else resolve(result); - }; - const stop = () => { - if (stopping || closed) return; - stopping = true; - child.kill('SIGTERM'); - killTimer = setTimeout(() => { - if (!closed) child.kill('SIGKILL'); - }, 5000); - }; - const failAndStop = (error) => { - stop(); - finish(error); - }; - const timer = setTimeout(() => { - timedOut = true; - stop(); - }, timeoutMs); - child.on('error', (error) => failAndStop(adapterFailure('Hermes', error?.message || 'spawn failed', { credentials }))); - child.stdout.on('data', (chunk) => { - if (settled) return; - buffer += decoder.write(chunk); - let newline = buffer.indexOf('\n'); - while (newline !== -1) { - if (newline > MAX_LINE_CHARS) { - failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); - return; - } - const line = buffer.slice(0, newline).trim(); - buffer = buffer.slice(newline + 1); - if (line) { - try { readHermesJsonl(line, state); } catch (error) { - failAndStop(error); - return; - } - } - newline = buffer.indexOf('\n'); - } - if (buffer.length > MAX_LINE_CHARS) { - failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); - } - }); - child.stdout.on('error', (error) => failAndStop(adapterFailure('Hermes stdout', error?.message || 'read failed', { credentials }))); - child.stderr.on('data', (chunk) => { - stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_CHARS); - }); - child.on('close', (code) => { - closed = true; - clearTimeout(killTimer); - if (settled) return; - if (timedOut) return finish(new Error(`Hermes timed out after ${timeoutMs}ms`)); - buffer += decoder.end(); - if (buffer.trim()) { - try { readHermesJsonl(buffer.trim(), state); } catch (error) { return finish(error); } - } - const result = state.result; - if (!result || !state.started) return finish(new Error('Hermes exited without a complete JSONL result')); - if (code !== 0 || result.exit_code !== 0) { - return finish(adapterFailure('Hermes', result.error || stderr || `exit ${code}`, { - credentials, exitCode: code, - })); - } - if (typeof result.text !== 'string' || !result.session_id) { - return finish(new Error('Hermes result omitted text or session_id')); - } - return finish(null, { text: result.text, newSessionId: result.session_id }); - }); - child.stdin.on('error', (error) => failAndStop(adapterFailure('Hermes stdin', error?.message || 'write failed', { credentials }))); - child.stdin.end(prompt); +import { seatModelRequired } from './seat-model.js'; + +export { TESTED_HERMES_VERSION, readHermesJsonl } from '@commonlyai/seats/adapters/hermes'; + +// Commonly's policy for the generic Hermes adapter: its memory preamble, its +// legacy sandbox-trust names, the fleet seat-model switch, and the profile +// directory seats have always used. +export default createHermesAdapter({ + memoryPreamble: buildMemoryPreamble, + effectiveTrust: effectiveSandboxTrust, + seatModelRequired, + profileRoot: join(homedir(), '.commonly', 'hermes', 'profiles'), }); - -export default { - get requiresModel() { return seatModelRequired(); }, - name: 'hermes', - runtimeType: 'hermes', - supportsProvider: false, - providerKeyEnv: 'OPENROUTER_API_KEY', - - async validateEnvironment(environment) { - if (environment?.provider !== undefined) { - throw new Error('Hermes adapter does not consume environment.provider; supply OPENROUTER_API_KEY in the host environment'); - } - if (environment?.mcp?.length) { - throw new Error('Hermes adapter does not yet consume environment.mcp declarations'); - } - if (effectiveSandboxTrust(environment?.sandbox?.trust) === 'public' - || (environment?.sandbox?.mode && environment.sandbox.mode !== 'none')) { - throw new Error('Hermes public or sandbox-declared seats are unsupported: no enforced confinement adapter exists'); - } - }, - - async detect() { - const version = readVersion(process.env); - return version ? { path: 'hermes', version } : null; - }, - - async spawn(prompt, ctx = {}) { - const seatModel = requireSeatModel(ctx.environment, 'hermes'); - await this.validateEnvironment(ctx.environment); - // Keep Hermes away from the operator's default profile, including any - // Nous Portal/Cloud credentials there. The wrapper provides the sole key. - const seat = `${ctx.agentName || ''}\0${ctx.cwd || ''}`; - const home = ctx._hermesHome || join(homedir(), '.commonly', 'hermes', 'profiles', - createHash('sha256').update(seat).digest('hex').slice(0, 24)); - await mkdir(home, { recursive: true, mode: 0o700 }); - const env = { ...(ctx.env || process.env), HERMES_HOME: home }; - if (!readVersion(env, ctx._spawnSyncImpl)) { - throw new Error('Hermes Agent 0.22.2 is not on PATH'); - } - if (!env.OPENROUTER_API_KEY) { - throw new Error('Supply the operator OPENROUTER_API_KEY through Secret Manager before running Hermes'); - } - const args = ['chat', '--oneshot', '--query-file', '-', '--format', 'stream-json', - '--provider', 'openrouter', '--source', 'tool', '--no-restore-cwd']; - if (seatModel) args.push('--model', seatModel); - if (ctx.sessionId) args.push('--resume', ctx.sessionId); - const fullPrompt = buildMemoryPreamble(prompt, ctx.memoryLongTerm, { - freshSession: !ctx.sessionId, - }); - const requestedTimeout = Number(ctx.timeoutMs); - const timeoutMs = Number.isFinite(requestedTimeout) && requestedTimeout > 0 - ? Math.min(requestedTimeout, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS; - return runHermes({ - args, prompt: fullPrompt, cwd: ctx.cwd, env, timeoutMs, - spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args, { modelFlag: '--model', valueFlags: ['--query-file', '--format', '--provider', '--source', '--resume'] }), - credentials: spawnCredentials(ctx, ['OPENROUTER_API_KEY']), - }); - }, -}; diff --git a/cli/src/lib/adapters/seat-model.js b/cli/src/lib/adapters/seat-model.js index fd9a0a7dd9..75c226d4a4 100644 --- a/cli/src/lib/adapters/seat-model.js +++ b/cli/src/lib/adapters/seat-model.js @@ -1,15 +1,8 @@ +import { requireSeatModel as requireDeclaredModel } from '@commonlyai/seats/seat-model'; + /** Fleet runners opt in; BYO users retain their own CLI's model default. */ export const seatModelRequired = () => process.env.COMMONLY_REQUIRE_SEAT_MODEL === '1'; -export const requireSeatModel = (environment, adapterName, required = seatModelRequired()) => { - const model = environment?.model; - if (typeof model !== 'string' || !model.trim()) { - if (!required) return undefined; - const error = new Error( - `${adapterName} requires environment.model; set the seat model with commonly agent config --model before running`, - ); - error.code = 'seat_model_missing'; - throw error; - } - return model.trim(); -}; +export const requireSeatModel = (environment, adapterName, required = seatModelRequired()) => ( + requireDeclaredModel(environment, adapterName, required) +); diff --git a/cli/src/lib/adapters/stub.js b/cli/src/lib/adapters/stub.js index 9cae043466..5e756a1b38 100644 --- a/cli/src/lib/adapters/stub.js +++ b/cli/src/lib/adapters/stub.js @@ -1,24 +1 @@ -/** - * Stub adapter — the no-op reference implementation of the ADR-005 adapter - * contract. Used by `commonly agent attach stub` and the Phase-1a test - * harness so the run loop can be reviewed end-to-end without a real CLI - * on PATH. - * - * Contract (ADR-005 §Adapter pattern): - * detect(): Promise<{ path, version } | null> - * spawn(prompt, ctx): Promise<{ text, newSessionId?, memorySummary? }> - * - * The adapter MUST be pure: input (argv + env + prompt) → output (text + optional - * next session-id + optional memory summary). No direct network, no direct API - * calls, no hidden state. The run loop handles all Commonly-facing I/O. - */ - -export default { - name: 'stub', - async detect() { - return { path: '(builtin)', version: '0.0.0' }; - }, - async spawn(prompt, _ctx) { - return { text: `(stub) received: ${String(prompt ?? '').slice(0, 200)}` }; - }, -}; +export { default } from '@commonlyai/seats/adapters/stub'; diff --git a/cli/src/lib/runtime-observation.js b/cli/src/lib/runtime-observation.js index 2dc35fbfcb..9fd70c5530 100644 --- a/cli/src/lib/runtime-observation.js +++ b/cli/src/lib/runtime-observation.js @@ -1,30 +1,6 @@ -// Read only adapter-owned control arguments. Callers exclude prompt/stdin and -// provide every value-taking option, so a flag-shaped model/session value is -// never reinterpreted as a second option. Raw arguments never leave the runner. -export const controlsFromArguments = (args, { modelFlag, effortFlag, valueFlags = [], codex = false }) => { - const controls = { model: null, effort: null }; - const flags = new Set([modelFlag, effortFlag, ...valueFlags].filter(Boolean)); - for (let index = 0; index < args.length; index += 1) { - const flag = args[index]; - if (!flags.has(flag)) continue; - const value = args[++index]; - if (flag === modelFlag) controls.model = value || null; - if (flag === effortFlag) controls.effort = value || null; - if (codex && flag === '-c' && value?.startsWith('model_reasoning_effort=')) { - try { controls.effort = JSON.parse(value.slice('model_reasoning_effort='.length)); } catch { /* unknown encoding */ } - } - } - return controls; -}; - -// Called at the child's successful OS spawn event, not when a requested -// configuration is saved or when a turn is merely queued. Never send argv. -export const observeSpawn = (child, onSpawn, controls) => { - if (typeof onSpawn !== 'function') return; - child.once('spawn', () => { - try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }); } catch { /* diagnostic only */ } - }); -}; +// controlsFromArguments and observeSpawn live in the seats runtime package; +// the reporter below posts to a Commonly instance, so it stays in the CLI. +export { controlsFromArguments, observeSpawn } from '@commonlyai/seats/runtime-observation'; export const createRuntimeReporter = ({ client, harness, detect }) => { let queue = Promise.resolve(); diff --git a/cli/src/lib/seat-stop.js b/cli/src/lib/seat-stop.js new file mode 100644 index 0000000000..b650dab298 --- /dev/null +++ b/cli/src/lib/seat-stop.js @@ -0,0 +1,96 @@ +/** + * How `agent run` stops on a signal while supervised process groups may be + * live (see @commonlyai/seats/process-group). + * + * SIGTERM keeps the in-flight turn: the run loop stops polling and lets the + * turn finish. A launchd seat is SIGKILLed ExitTimeOut seconds after SIGTERM, + * and that SIGKILL cannot be caught, so a supervised group (outside the seat's + * own group, which launchd kills) would outlive the seat. So on a launchd + * seat, SIGTERM also arms a deadline a margin before ExitTimeOut, and the exit + * path kills whatever groups are left when the turn ends first. + * + * The deadline is armed only when the seat's plist exists. A seat without one + * (the daemon spawns it) is stopped with SIGTERM and never SIGKILLed, so it + * relies on the exit path alone, and a reassignment or config change does not + * cut the in-flight turn. + * + * ExitTimeOut is read from the seat's plist and never written. In a plist that + * exists, an unset, zero or unreadable value is launchd's default of 20 s, + * because launchd will SIGKILL that seat. + */ +import { spawnSync } from 'child_process'; +import { existsSync, readFileSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; +import { armSupervisedGroupDeadline, killSupervisedGroups } from '@commonlyai/seats/process-group'; + +export const DEFAULT_EXIT_TIMEOUT_S = 20; +const MAX_MARGIN_MS = 5000; +const SEAT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +/** The seat's launchd plist path, or null for a name that cannot be a file name. */ +export const seatPlistPath = (name, home = homedir()) => (typeof name === 'string' && SEAT_NAME.test(name) + ? join(home, 'Library', 'LaunchAgents', `me.commonly.seat-${name}.plist`) + : null); + +const positive = (value) => (Number.isFinite(value) && value > 0 ? value : null); + +/** + * ExitTimeOut in seconds, or null when the plist is missing, has none, or + * holds a value launchd would not use as a timeout. An XML plist is read + * directly; a binary one through `plutil -extract` (read-only). + */ +export const readExitTimeOut = (path, { readFile = readFileSync, run = spawnSync } = {}) => { + if (!path) return null; + let raw; + try { raw = readFile(path); } catch { return null; } + const text = Buffer.isBuffer(raw) ? raw.toString('utf8') : String(raw); + if (text.startsWith('bplist')) { + try { + const out = run('plutil', ['-extract', 'ExitTimeOut', 'raw', '-o', '-', path], { encoding: 'utf8', timeout: 5000 }); + return out.status === 0 ? positive(Number(String(out.stdout).trim())) : null; + } catch { return null; } + } + const match = /\s*ExitTimeOut\s*<\/key>\s*<(integer|real)>\s*([0-9.]+)\s*<\/\1>/.exec(text); + return match ? positive(Number(match[2])) : null; +}; + +/** + * Whether the seat's plist exists, and its ExitTimeOut (null when unset or + * unreadable). Only an existing plist means launchd will SIGKILL this seat. + */ +export const readSeatPlist = (path, { exists = existsSync, ...deps } = {}) => { + if (!path || !exists(path)) return { present: false, exitTimeOut: null }; + return { present: true, exitTimeOut: readExitTimeOut(path, deps) }; +}; + +/** Milliseconds after SIGTERM to kill the groups: ExitTimeOut less a margin of 5 s or a quarter, whichever is less. */ +export const groupKillDeadlineMs = (exitTimeOutS) => { + const totalMs = (positive(exitTimeOutS) ?? DEFAULT_EXIT_TIMEOUT_S) * 1000; + return Math.floor(totalMs - Math.min(MAX_MARGIN_MS, totalMs / 4)); +}; + +/** + * `requestStop(signal)` stops the run loop and, on SIGTERM to a seat with a + * plist, arms the group deadline; `onExit` kills the groups left at exit. + * `deps` is a test seam. + */ +export const createSeatStop = ({ name, stop, home, deps = {} } = {}) => { + const { + arm = armSupervisedGroupDeadline, + kill = killSupervisedGroups, + readPlist = readSeatPlist, + } = deps; + return { + requestStop(signal) { + stop(); + if (signal !== 'SIGTERM') return null; + const plist = readPlist(seatPlistPath(name, home)); + if (!plist.present) return null; + const deadlineMs = groupKillDeadlineMs(plist.exitTimeOut); + arm(deadlineMs); + return deadlineMs; + }, + onExit() { kill(); }, + }; +}; diff --git a/cli/src/lib/upstream-refusal.js b/cli/src/lib/upstream-refusal.js index 598edb8a51..f70cef6312 100644 --- a/cli/src/lib/upstream-refusal.js +++ b/cli/src/lib/upstream-refusal.js @@ -1,354 +1 @@ -/** - * TASK-096 — name a refused model route, instead of reporting the silence it - * causes. - * - * Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape - * that made a seat look broken for two and a half days): pi runs its own retry - * ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and - * EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only - * `type: "error"` events — of which this produces none — so the wrapper logged - * `no wrapper-post (empty output)` and the cause sat unread in the stream. - * - * Two rules govern the text that reaches a seat log: - * - * 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes - * only the shapes we thought of, and a 401 can echo the key back. So a body - * contributes one NAMED field or nothing at all. - * 2. Exact-match refusal for credentials. If the kept text contains a value - * this spawn was handed, the text is dropped — a known value, not a guessed - * shape (Wren 70877). - * - * Two limits follow from rule 2. They are the accepted cost of refusing guessed - * shapes, named here so they are not rediscovered as defects later (Vera 71082): - * - * a. The match is exact and against THIS spawn's values only, so a credential - * that arrives transformed — base64 inside a proxy error, percent-encoded in - * a URL echo — does not match and the text is kept. Matching transformed - * shapes instead would reintroduce the class rule 1 exists to avoid. - * b. The keep-list can still surface someone ELSE's secret. A third party's key - * relayed in an upstream error is not detectable here, because the one - * inventory this module has is the set of values it was itself handed. - * - * Scope: this reader was wired into the pi adapter only. TASK-103 extended it - * to codex and claude, which are NOT silent the way pi was: the claude adapter - * rejects on a non-zero exit and deliberately reports stdout with it, because a - * `-p` run writes quota conditions there; the codex adapter rejects on - * `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text - * they raised was the raw tail, unscrubbed by the rules above, and carried no - * HTTP status for the run loop to classify on. `scrubAdapterFailure` below is - * the shared reader they now both go through, so the two rules cannot drift - * apart between three adapters. - */ - -/** The kept detail is truncated, ellipsis included, to this many characters. */ -export const MAX_REFUSAL_DETAIL = 256; - -/** - * The status an adapter prefixes to its error text: `"429: {…}"` -> `429`. - * Null for anything that is not a 4xx/5xx, so a message that merely starts with - * a number ("4040 tokens over budget") is not read as a status. - * - * The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1 - * emits `"429: {"message":…}"` for a JSON body and `"502 …"` for an - * HTML one. Both forms are matched here; a parser that required the colon would - * have kept `502 …` as the whole "body" and then dropped it as non-JSON, - * reporting the status alone with no sign that the body was never read. - */ -const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/; - -export const statusFromError = (value) => { - const match = STATUS_PREFIX.exec(String(value ?? '')); - if (!match) return null; - const status = Number(match[1]); - return status >= 400 && status <= 599 ? status : null; -}; - -/** - * The body the adapter carried after its status prefix, or the whole text when - * there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`; - * `"502 "` -> ``. - */ -export const bodyFromError = (value) => { - const text = String(value ?? ''); - if (statusFromError(text) === null) return text.trim(); - return text.replace(STATUS_PREFIX, '').trim(); -}; - -/** - * Replace credential values this spawn was handed with a marker. - * - * This is the exact-match half, for text that is kept for its whole shape rather - * than filtered to one field — an adapter's stderr tail, say, whose diagnostic - * value is in the lines around the secret. Values shorter than eight characters - * are ignored: redacting a one-character "credential" would corrupt ordinary - * prose, and no issued credential is that short. - */ -export const redactKnownCredentials = (text, credentials = []) => { - let out = String(text ?? ''); - for (const value of credentials) { - if (typeof value !== 'string' || value.length < 8) continue; - out = out.split(value).join('[redacted]'); - } - return out; -}; - -/** - * The keep-list. Given the raw body an adapter captured, keep at most one named - * field of it and only when that field is a string: - * - * `error.message` first (the OpenAI-shaped error), then a top-level `message` - * (which is the shape LiteLLM's budget refusal actually uses — the motivating - * case; a keep-list that kept only `error.message` would name the status and - * still hide the budget line that started this row). - * - * Everything else — HTML, a non-JSON body, a JSON body with neither field, a - * body echoing a known credential — contributes nothing. Returns null rather - * than a partial guess. - */ -export const keptRefusalDetail = (raw, { credentials = [] } = {}) => { - let parsed; - try { - parsed = JSON.parse(String(raw ?? '')); - } catch { - return null; - } - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; - - const candidate = parsed?.error?.message ?? parsed?.message; - if (typeof candidate !== 'string') return null; - - const text = candidate.replace(/\s+/g, ' ').trim(); - if (!text) return null; - - const redacted = redactKnownCredentials(text, credentials); - // `[redacted]` appearing is the signal that the body echoed something this - // spawn was handed. The status is still worth keeping; the text is not. - if (redacted !== text) return null; - - return text.length > MAX_REFUSAL_DETAIL - ? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…` - : text; -}; - -/** - * Read a pi JSON stream and return the refusal it ended on, or null. - * - * `auto_retry_end` is the terminal word — it names the status and the body after - * the ladder has finished — so it wins over the per-attempt `message_end` - * events that precede it. A run that refused on its first attempt and never - * retried still reports through `message_end`. - */ -export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => { - let status = null; - let raw = null; - - for (const line of String(stdout ?? '').split('\n')) { - if (!line.trim()) continue; - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - - if (event?.type === 'auto_retry_end' && event.success === false) { - const candidate = event.finalError ?? event.errorMessage; - const attemptStatus = statusFromError(candidate); - if (attemptStatus) { - status = attemptStatus; - raw = bodyFromError(candidate); - } - continue; - } - - const message = event?.message; - if ( - event?.type === 'message_end' - && message?.role === 'assistant' - && message.stopReason === 'error' - && status === null - ) { - const attemptStatus = statusFromError(message.errorMessage); - if (attemptStatus) { - status = attemptStatus; - raw = bodyFromError(message.errorMessage); - } - } - } - - if (status === null) return null; - return { status, detail: keptRefusalDetail(raw, { credentials }) }; -}; - -/** - * Is this text STRUCTURED upstream data — a JSON object or an array? The gate for - * the whole-tail fallback below, deliberately NOT `kept === null`, which is true - * for five different reasons (Vera 71350). - * - * An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is - * structured data with fields this code does not know, which is exactly what rule - * 1 exists for, and treating it as prose shipped the whole body under exact-match - * alone. A JSON *string* is the opposite case and does NOT count — `"Too many - * requests"` is a whole-shape text with no fields to reduce, so it keeps its - * wording like any other prose. - */ -const isStructuredBody = (raw) => { - try { - const parsed = JSON.parse(String(raw ?? '')); - return parsed !== null && typeof parsed === 'object'; - } catch { - return false; - } -}; - -/** - * A 3-digit number is read as a status only when the tail NAMES it as one: - * `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | ` - * segment followed by a colon, which is the shape claude's `stderr | stdout` - * join produces for a body that leads with its status. - * - * A bare `\b(\d{3})\b` scan is not enough, and the false positives are not - * hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds` - * all read as 500, and `classifySpawnFailure` tests the status FIELD before the - * message — so a count misread as a status stands the fleet down on a refusal - * that never happened, which is worse than no status at all (Vera 71352). - */ -const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi; -// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is, -// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED -// line: the `1` in `/1.1` is a -// WORD character, so `HTTP` and `429` are not within four non-word characters of -// each other. It read as `status: null` — an unclassified runtime failure on the -// single most recognisable shape, which is the misclassification this row exists -// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap -// would start reading `/v1/models 500` as a status. -const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi; -const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g; - -/** The first 4xx/5xx one of those shapes names, or null. */ -const scanStatuses = (text, pattern) => { - for (const match of String(text ?? '').matchAll(pattern)) { - const value = Number(match[1]); - if (value >= 400 && value <= 599) return value; - } - return null; -}; - -/** - * Find the status AND the body boundary, because they are the same question. - * - * Anchoring the status to a ` | ` segment start is not enough on its own: the - * body has to be taken from just past that status too, or a mid-tail read - * isolates the status and then hands the WHOLE joined tail to the body rules, - * which no longer look like a JSON body and so fall through to "keep it all". - * That is the difference between dropping a credential-echoing body and shipping - * the part of it that exact-match cannot see. - * - * `STATUS_COLON` is the only shape that moves the boundary, because it is the - * same shape as the anchored prefix (`429: {body}`), just found at a segment - * boundary — claude's `stderr | stdout` join. A status that appears inside a - * body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status - * (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body. - */ -const readStatusAndBody = (text) => { - const raw = String(text ?? ''); - const prefixed = statusFromError(raw); - if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) }; - // Only the colon shape moves the boundary — see the note above. A status LINE - // names the status but its reason phrase is prose, so the body stays the whole - // text and the exact-match rule protects it. - for (const match of raw.matchAll(STATUS_COLON)) { - const value = Number(match[1]); - if (value >= 400 && value <= 599) { - return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') }; - } - } - const named = scanStatuses(raw, STATUS_NAMED); - return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw }; -}; - -/** - * The shared reader for an adapter's own failure tail (TASK-103). - * - * The two rules above are about a body an adapter captured, and they apply here - * too — but the tail is a different shape from pi's refusal, and the difference - * decides how each rule lands: - * - * - The tail is often NOT structured. Claude's `-p` mode writes its terminal - * condition as prose (`Claude usage limit reached. Your limit will reset at - * 11:40pm.`), and that line is the whole diagnostic value of reporting - * stdout at all — 361 consecutive failures on 2026-08-03 carried no reason - * because of it. So the keep-list applies only when the tail parses as a - * body; when it does not, the text is KEPT as a whole shape and the - * exact-match rule is what protects it (`redactKnownCredentials`, which - * substitutes rather than drops, precisely because a whole-shape text has no - * second field to fall back to). - * - * - A body that IS structured (a JSON object or array) is reduced to one named - * field, and a body whose field echoed a value this spawn was handed is - * DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the - * whole tail is gated on the SHAPE (is it structured at all), never on - * "the keep-list returned null": that null has - * four other causes, and one of them is the credential echo this line exists - * to drop. Gating on `kept === null` would emit the whole body for that case, - * which is precisely what rule 2 forbids — and exact-match cannot see a - * transformed shape (limit (a) in the header), so a key that arrived - * base64'd or percent-encoded would ship (Vera 71350/71351). - * - * Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null. - * The caller decides where the status goes; `classifySpawnFailure` reads it off - * `error.status` or out of the message text, so both a leading `429: …` prefix - * and an attached field classify the same way. `detail` is `''` when nothing of - * the body may be shown — the status is then the whole diagnostic. - */ -export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => { - const raw = String(text ?? '').trim(); - const { status, body } = readStatusAndBody(raw); - const kept = keptRefusalDetail(body, { credentials }); - const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials)); - return { status, detail: detail.slice(0, limit) }; -}; - -/** - * The Error an adapter raises, built from its own failure tail. - * - * `status` is attached as a field, not only written into the text: the run - * loop's classifier tests `error.status === 429` before it looks at the - * message, and a refusal that is only readable by regex is one wording away - * from the misclassification that cost an hour on 2026-08-18 (see - * spawn-retry.js). The message still names the status so a human log says the - * same thing as the classifier's decision. - */ -export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => { - const { status, detail } = scrubAdapterFailure(text, { credentials, limit }); - const code = exitCode === null ? '' : ` exited with code ${exitCode}`; - const named = status === null ? '' : ` (upstream ${status})`; - const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`); - if (status !== null) error.status = status; - return error; -}; - -/** - * The seat-log clause. The status is always present; the detail only when the - * keep-list kept one. `grep 'upstream refused'` finds every refusal, which is - * the property the old `no wrapper-post (empty output)` line did not have. - */ -export const describeUpstreamRefusal = (refusal) => { - if (!refusal || typeof refusal.status !== 'number') return null; - return refusal.detail - ? `upstream refused ${refusal.status}: ${refusal.detail}` - : `upstream refused ${refusal.status}`; -}; - -/** - * The credential values this spawn was handed, for the exact-match check in - * `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the - * caller supplies them because only the adapter knows which key its route uses. - */ -export const spawnCredentials = (ctx = {}, envNames = []) => { - const env = ctx.env || process.env || {}; - return [ - ctx.runtimeToken, - ...envNames.map((name) => env[name]), - ].filter((value) => typeof value === 'string' && value.length >= 8); -}; +export * from '@commonlyai/seats/upstream-refusal'; diff --git a/packages/seats/README.md b/packages/seats/README.md new file mode 100644 index 0000000000..990cbab2c0 --- /dev/null +++ b/packages/seats/README.md @@ -0,0 +1,11 @@ +# packages/seats + +The adapters that run an agent CLI as a supervised seat: one prompt in, one CLI run, one reply out. The package holds the contract, the registry, the failure naming, the spawn observation, the run supervisor (`superviseRun`: a turn with no progress signal for `stallMs` is traced as stuck, cancelled, and settled even if its adapter ignores the cancel), and the first adapters (`stub`, `hermes`). + +It is private and is not published on its own. `@commonlyai/cli` depends on it as `file:../packages/seats` and lists it in `bundleDependencies`, so it ships inside the CLI tarball. + +**Rule:** nothing here imports Commonly code. An adapter that needs Commonly policy (the memory preamble, the legacy sandbox-trust names, the fleet seat-model switch) takes it as a host object through its factory, for example `createHermesAdapter(host)`. The CLI binds that host in `cli/src/lib/adapters/.js`. `cli/__tests__/seats-package.test.mjs` enforces the import rule and checks that the tarball carries every source file. + +The extraction plan is D62, extract-in-place. + +**Run record.** The trace store writes each run as a versioned, hash-chained JSONL record. It covers who granted which agent what access for which task, who approved each call, what was attempted, and the result. A tool outside this package can read, verify and export it to OTLP. The format is specified in [`docs/record-v1.md`](docs/record-v1.md), and a generated sample is in [`docs/examples/`](docs/examples/). diff --git a/packages/seats/docs/examples/TASK-101.otlp.json b/packages/seats/docs/examples/TASK-101.otlp.json new file mode 100644 index 0000000000..953d0ea88f --- /dev/null +++ b/packages/seats/docs/examples/TASK-101.otlp.json @@ -0,0 +1,2507 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "seats-sample" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "seats.record", + "version": "1" + }, + "spans": [ + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "4f7ea2299f50ef90", + "name": "seats.run", + "kind": 1, + "startTimeUnixNano": "1792944002000000000", + "endTimeUnixNano": "1792944031000000000", + "attributes": [ + { + "key": "seats.run.id", + "value": { + "stringValue": "run-ana-0001" + } + }, + { + "key": "seats.task.id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.pod.id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.run.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "seats.run.attempts", + "value": { + "intValue": "2" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944002000000000", + "name": "task.claimed", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.lease_until", + "value": { + "stringValue": "2026-10-25T18:00:00Z" + } + } + ] + }, + { + "timeUnixNano": "1792944003000000000", + "name": "grant.issued", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.binding.ends_on", + "value": { + "stringValue": "task.done" + } + }, + { + "key": "seats.binding.pod_id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.binding.task_id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.expires_at", + "value": { + "stringValue": "2026-10-25T20:00:00.000Z" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.issued_by", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.parent_grant_id", + "value": { + "stringValue": "grant-ben-sentry-root" + } + }, + { + "key": "seats.scope.audience", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "u-ana" + }, + { + "stringValue": "u-ben" + } + ] + } + } + }, + { + "key": "seats.scope.budget.calls", + "value": { + "intValue": "20" + } + }, + { + "key": "seats.scope.budget.window_ms", + "value": { + "intValue": "3600000" + } + }, + { + "key": "seats.scope.connection_id", + "value": { + "stringValue": "conn-ben-sentry" + } + }, + { + "key": "seats.scope.tools", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "sentry.get_issue" + }, + { + "stringValue": "sentry.resolve_issue" + } + ] + } + } + }, + { + "key": "seats.scope.write_mode", + "value": { + "stringValue": "write-with-confirm" + } + }, + { + "key": "seats.target.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.target.kind", + "value": { + "stringValue": "seat" + } + } + ] + }, + { + "timeUnixNano": "1792944004000000000", + "name": "credential.granted", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "3" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.names", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "OPENROUTER_API_KEY" + }, + { + "stringValue": "SENTRY_PROXY_TOKEN" + } + ] + } + } + } + ] + }, + { + "timeUnixNano": "1792944012000000000", + "name": "decision.ruled", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "11" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "agent-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-2b9" + } + }, + { + "key": "seats.value", + "value": { + "stringValue": "retry" + } + } + ] + }, + { + "timeUnixNano": "1792944013000000000", + "name": "liveness.recovery", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "12" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.action", + "value": { + "stringValue": "retry" + } + }, + { + "key": "seats.attempt", + "value": { + "intValue": "2" + } + } + ] + }, + { + "timeUnixNano": "1792944030000000000", + "name": "result.reported", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "29" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.refs", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "id", + "value": { + "stringValue": "#212" + } + }, + { + "key": "type", + "value": { + "stringValue": "pr" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "done" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "f213e9d395dc7277", + "parentSpanId": "4f7ea2299f50ef90", + "name": "invoke_agent Ana's opencode", + "kind": 1, + "startTimeUnixNano": "1792944005000000000", + "endTimeUnixNano": "1792944011000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.19.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "stuck" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "stuck" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944006000000000", + "name": "turn.prompt_sent", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "5" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.chars", + "value": { + "intValue": "412" + } + } + ] + }, + { + "timeUnixNano": "1792944009000000000", + "name": "liveness.stalled", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "8" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.idle_ms", + "value": { + "intValue": "90000" + } + }, + { + "key": "seats.window_ms", + "value": { + "intValue": "90000" + } + } + ] + }, + { + "timeUnixNano": "1792944010000000000", + "name": "liveness.recovery", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "9" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.action", + "value": { + "stringValue": "cancel" + } + }, + { + "key": "seats.attempt", + "value": { + "intValue": "1" + } + } + ] + } + ], + "status": { + "code": 2, + "message": "stuck" + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "a95791040c71e9f0", + "parentSpanId": "f213e9d395dc7277", + "name": "execute_tool Read src/checkout/total.js", + "kind": 1, + "startTimeUnixNano": "1792944007000000000", + "endTimeUnixNano": "1792944008000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "Read src/checkout/total.js" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-1" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "native" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944008000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "7" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "109cfe1b8a44b804", + "parentSpanId": "4f7ea2299f50ef90", + "name": "invoke_agent Ana's opencode", + "kind": 1, + "startTimeUnixNano": "1792944014000000000", + "endTimeUnixNano": "1792944029000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "ses-ana-2" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.19.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "18234" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "2210" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944028000000000", + "name": "usage", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "27" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.input_tokens", + "value": { + "intValue": "18234" + } + }, + { + "key": "seats.output_tokens", + "value": { + "intValue": "2210" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "2f16f44fc7fa7d44", + "parentSpanId": "109cfe1b8a44b804", + "name": "execute_tool git push origin fix/total-rounding", + "kind": 1, + "startTimeUnixNano": "1792944015000000000", + "endTimeUnixNano": "1792944018000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "git push origin fix/total-rounding" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-2" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "execute" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "native" + } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "tool_failed" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944016000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "15" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.question", + "value": { + "stringValue": "git push origin fix/total-rounding" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + }, + { + "timeUnixNano": "1792944017000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "16" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "reject" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "deny" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "execute:git push*" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + }, + { + "timeUnixNano": "1792944018000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "17" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "failed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + } + ], + "status": { + "code": 2, + "message": "tool_failed" + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "e9bb6dcc7812bd2c", + "parentSpanId": "109cfe1b8a44b804", + "name": "execute_tool sentry.resolve_issue", + "kind": 1, + "startTimeUnixNano": "1792944019000000000", + "endTimeUnixNano": "1792944022000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "sentry.resolve_issue" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-3" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "fetch" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "connector" + } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944020000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "19" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "credential-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-7f3" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.question", + "value": { + "stringValue": "Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + }, + { + "timeUnixNano": "1792944021000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "20" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.approval", + "value": { + "stringValue": "approved" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "credential-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-7f3" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "ask" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "fetch" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + }, + { + "timeUnixNano": "1792944022000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "21" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "d54dc8e7e5d85bf5", + "parentSpanId": "109cfe1b8a44b804", + "name": "execute_tool commonly_post_message", + "kind": 1, + "startTimeUnixNano": "1792944023000000000", + "endTimeUnixNano": "1792944026000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "commonly_post_message" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-4" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "other" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "commonly" + } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944024000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "23" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.question", + "value": { + "stringValue": "commonly_post_message: the fix summary, to the pod" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + }, + { + "timeUnixNano": "1792944025000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "24" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "other:commonly_*" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + }, + { + "timeUnixNano": "1792944026000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "25" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "a4aa63e7490207ea", + "name": "seats.run", + "kind": 1, + "startTimeUnixNano": "1792944033000000000", + "endTimeUnixNano": "1792944044000000000", + "attributes": [ + { + "key": "seats.run.id", + "value": { + "stringValue": "run-ben-0001" + } + }, + { + "key": "seats.task.id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.pod.id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.run.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "seats.run.attempts", + "value": { + "intValue": "1" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944033000000000", + "name": "grant.issued", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.binding.ends_on", + "value": { + "stringValue": "task.done" + } + }, + { + "key": "seats.binding.pod_id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.binding.task_id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.expires_at", + "value": { + "stringValue": "2026-10-25T20:00:00.000Z" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, + { + "key": "seats.issued_by", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.scope.audience", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "u-ben" + } + ] + } + } + }, + { + "key": "seats.scope.connection_id", + "value": { + "stringValue": "conn-ben-github" + } + }, + { + "key": "seats.scope.tools", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "github.get_pull_request" + } + ] + } + } + }, + { + "key": "seats.scope.write_mode", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.target.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.target.kind", + "value": { + "stringValue": "seat" + } + } + ] + }, + { + "timeUnixNano": "1792944034000000000", + "name": "credential.granted", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.names", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "OPENAI_API_KEY" + } + ] + } + } + } + ] + }, + { + "timeUnixNano": "1792944042000000000", + "name": "task.done", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "10" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "board" + } + }, + { + "key": "seats.done_by", + "value": { + "stringValue": "u-ben" + } + } + ] + }, + { + "timeUnixNano": "1792944043000000000", + "name": "grant.revoked", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "11" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "board" + } + }, + { + "key": "seats.grant_ids", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "grant-ben-review-1" + } + ] + } + } + }, + { + "key": "seats.reason", + "value": { + "stringValue": "task.done" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "5fd2950fcbddd334", + "parentSpanId": "a4aa63e7490207ea", + "name": "invoke_agent Ben's goose", + "kind": 1, + "startTimeUnixNano": "1792944035000000000", + "endTimeUnixNano": "1792944041000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "goose" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "goose" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.54.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "ok" + } + } + ], + "events": [], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "d45b1520df0f2e72", + "parentSpanId": "5fd2950fcbddd334", + "name": "execute_tool github.get_pull_request", + "kind": 1, + "startTimeUnixNano": "1792944036000000000", + "endTimeUnixNano": "1792944039000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "github.get_pull_request" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-1" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "connector" + } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944037000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "5" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ben-review-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.question", + "value": { + "stringValue": "Read pull request #212" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + }, + { + "timeUnixNano": "1792944038000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "6" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ben-review-1" + } + }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + }, + { + "timeUnixNano": "1792944039000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "7" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + } + ], + "status": { + "code": 0 + } + } + ] + } + ] + } + ] +} diff --git a/packages/seats/docs/examples/run-ana-0001.jsonl b/packages/seats/docs/examples/run-ana-0001.jsonl new file mode 100644 index 0000000000..05afbb7280 --- /dev/null +++ b/packages/seats/docs/examples/run-ana-0001.jsonl @@ -0,0 +1,30 @@ +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"leaseUntil":"2026-10-25T18:00:00Z"},"kind":"task.claimed","podId":"pod-checkout","prev":null,"runId":"run-ana-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:02.000Z","v":1} +{"actor":{"id":"u-ana","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ana-sentry-1","issuedBy":"u-ana","parentGrantId":"grant-ben-sentry-root","scope":{"audience":["u-ana","u-ben"],"budget":{"calls":20,"windowMs":3600000},"connectionId":"conn-ben-sentry","tools":["sentry.get_issue","sentry.resolve_issue"],"writeMode":"write-with-confirm"},"target":{"id":"u-agent-ana-opencode","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":"112fd59d4b22a34212a1051217ea50cbee7fc5756d1db32c0487760e84e0b2c2","runId":"run-ana-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:03.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENROUTER_API_KEY","SENTRY_PROXY_TOKEN"]},"kind":"credential.granted","podId":"pod-checkout","prev":"02b60a920d4e46710f1da5d478472e32054abef954f1921f836cfb7b8f2466b2","runId":"run-ana-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:04.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":1,"cli":{"name":"opencode","version":"1.19.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"8de2801011b74ef1386e35eaa53bf6e0d5fce3692309ba832bc44a9005c2b97c","runId":"run-ana-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:05.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"chars":412},"kind":"turn.prompt_sent","podId":"pod-checkout","prev":"2191f906c453d8adbf118b25a096134313c4db9f2776c9d81462eef066c80a88","runId":"run-ana-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:06.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"read","status":"pending","surface":"native","title":"Read src/checkout/total.js","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"53d2f32e2e78ce199c9b84eba5140d79b62b01f2ef8de43acd527e98739d43a3","runId":"run-ana-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:07.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"40c3dfe7e66cec5306e69ed96798950b8d1a1aa8e1dd6ad8c8384439b99f7ce3","runId":"run-ana-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:08.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"idleMs":90000,"windowMs":90000},"kind":"liveness.stalled","podId":"pod-checkout","prev":"212b5d465b8936032324127c8b834523df956bc6bc5fde15004d0b4932009dfb","runId":"run-ana-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:09.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"action":"cancel","attempt":1},"kind":"liveness.recovery","podId":"pod-checkout","prev":"260faaf14f1c7bf121338f251a1c88e41c78689ccdbfa0fe3f36831bda5c3848","runId":"run-ana-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:10.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"failure":{"detail":"no progress signal for 90000ms (window 90000ms)","kind":"stuck"},"outcome":"stuck"},"kind":"attempt.settled","podId":"pod-checkout","prev":"c7637334d2a2bdeea8d5d6ae3b402d6d348bcd18ee563cbe2ef00576cb7c44ad","runId":"run-ana-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:11.000Z","v":1} +{"actor":{"id":"u-ana","type":"human"},"data":{"decidedBy":"u-ana","decidedByRole":"agent-owner","decisionCardId":"card-2b9","value":"retry"},"kind":"decision.ruled","podId":"pod-checkout","prev":"0ee1be8a38ec355ebd500c99c09517e2607ab01a61ef6989770bb09a1b1f3adf","runId":"run-ana-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:12.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"action":"retry","attempt":2},"kind":"liveness.recovery","podId":"pod-checkout","prev":"440445ac0d77d87d42c2bcb22b5efa55331e923f0e36b3447dd9a9a64ece95b2","runId":"run-ana-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:13.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":2,"cli":{"name":"opencode","version":"1.19.0"},"sessionId":"ses-ana-2"},"kind":"attempt.started","podId":"pod-checkout","prev":"463755acfb130eda7a974c6be8d4af9a080556d5e103ecd16d90bdb77e2e7f43","runId":"run-ana-0001","schema":"seats.record","seq":13,"taskId":"TASK-101","ts":"2026-10-25T16:00:14.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","kind":"execute","status":"pending","surface":"native","title":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"tool.call","podId":"pod-checkout","prev":"8370cccac6497cee039bda506eaabbaaa6e81c3067520062d6bb91329853e41e","runId":"run-ana-0001","schema":"seats.record","seq":14,"taskId":"TASK-101","ts":"2026-10-25T16:00:15.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","askedOf":{"id":"grant:grant-ana-sentry-1","role":"policy"},"grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"permission.requested","podId":"pod-checkout","prev":"ad5435f5ed0c1d500e8c2ddaec7b8023d72660d9daea5f5c8c57266fc48c8e63","runId":"run-ana-0001","schema":"seats.record","seq":15,"taskId":"TASK-101","ts":"2026-10-25T16:00:16.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"reject_once","optionId":"reject","outcome":"selected","rule":{"list":"deny","pattern":"execute:git push*"},"toolCallId":"tc-2"},"kind":"permission.decided","podId":"pod-checkout","prev":"bae28e9f22669c2f9b24668500e0faa57d193a936dc3240a6e2f655751180d20","runId":"run-ana-0001","schema":"seats.record","seq":16,"taskId":"TASK-101","ts":"2026-10-25T16:00:17.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"failed","toolCallId":"tc-2"},"kind":"tool.update","podId":"pod-checkout","prev":"d41a409cfd72534a80e4d08609daafbd0440a22c4964631ff99c49fbd4d13475","runId":"run-ana-0001","schema":"seats.record","seq":17,"taskId":"TASK-101","ts":"2026-10-25T16:00:18.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","kind":"fetch","name":"sentry.resolve_issue","status":"pending","surface":"connector","title":"Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212","toolCallId":"tc-3"},"kind":"tool.call","podId":"pod-checkout","prev":"bce5d5620503b5040aebc590e429962fd453ac6bb84648b7a3e725677749724f","runId":"run-ana-0001","schema":"seats.record","seq":18,"taskId":"TASK-101","ts":"2026-10-25T16:00:19.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","askedOf":{"id":"u-ben","role":"credential-owner"},"decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212","toolCallId":"tc-3"},"kind":"permission.requested","podId":"pod-checkout","prev":"cfa49b7b5892c724099e22752af46b6b6f78040886dc404019bcfedb48605206","runId":"run-ana-0001","schema":"seats.record","seq":19,"taskId":"TASK-101","ts":"2026-10-25T16:00:20.000Z","v":1} +{"actor":{"id":"u-ben","type":"human"},"data":{"approval":"approved","argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","decidedBy":"u-ben","decidedByRole":"credential-owner","decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"ask","pattern":"fetch"},"toolCallId":"tc-3"},"kind":"permission.decided","podId":"pod-checkout","prev":"e8b32872b12d59434da0832df0c96cc1d7c3fc5ca9cfc14612cb00c5af624b62","runId":"run-ana-0001","schema":"seats.record","seq":20,"taskId":"TASK-101","ts":"2026-10-25T16:00:21.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-3"},"kind":"tool.update","podId":"pod-checkout","prev":"2a277938e8fd32717316cc03f6f79d46034b405b3bf729157b80bb7852bf14f6","runId":"run-ana-0001","schema":"seats.record","seq":21,"taskId":"TASK-101","ts":"2026-10-25T16:00:22.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","kind":"other","name":"commonly_post_message","status":"pending","surface":"commonly","title":"commonly_post_message: the fix summary, to the pod","toolCallId":"tc-4"},"kind":"tool.call","podId":"pod-checkout","prev":"fc2b15b652bcb8f00463f829ba89a236220c55f8630db6f6d2b32834bd7ba25e","runId":"run-ana-0001","schema":"seats.record","seq":22,"taskId":"TASK-101","ts":"2026-10-25T16:00:23.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","askedOf":{"id":"grant:grant-ana-sentry-1","role":"policy"},"grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"commonly_post_message: the fix summary, to the pod","toolCallId":"tc-4"},"kind":"permission.requested","podId":"pod-checkout","prev":"23550e8771e7124cf2ffb378bf23cb015fc909da9ab30b38c71ab3d72b636379","runId":"run-ana-0001","schema":"seats.record","seq":23,"taskId":"TASK-101","ts":"2026-10-25T16:00:24.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"other:commonly_*"},"toolCallId":"tc-4"},"kind":"permission.decided","podId":"pod-checkout","prev":"7036a70b913d9b4c6e2a8cdcd965be61f3881b2c1fe8f74617d5b45c44a29e67","runId":"run-ana-0001","schema":"seats.record","seq":24,"taskId":"TASK-101","ts":"2026-10-25T16:00:25.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-4"},"kind":"tool.update","podId":"pod-checkout","prev":"08800755ccfa37befaf6d6eeb849d79e97514f1f8fdfa9a68594487e694da893","runId":"run-ana-0001","schema":"seats.record","seq":25,"taskId":"TASK-101","ts":"2026-10-25T16:00:26.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"text":"Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"f63d36617240b79419e99a85bd1d5376b3cc4d7c6a7670abc5bb9a57308a200f","runId":"run-ana-0001","schema":"seats.record","seq":26,"taskId":"TASK-101","ts":"2026-10-25T16:00:27.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"inputTokens":18234,"outputTokens":2210},"kind":"usage","podId":"pod-checkout","prev":"99a5c2a6d12f37d7ea10bc4aa0780f5ea3edcaeb10fed13757c8969f31364c6a","runId":"run-ana-0001","schema":"seats.record","seq":27,"taskId":"TASK-101","ts":"2026-10-25T16:00:28.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":2,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"a53827fe95252e39d8911788de097522424b66db9cd9ad717721c7ca1be7cb38","runId":"run-ana-0001","schema":"seats.record","seq":28,"taskId":"TASK-101","ts":"2026-10-25T16:00:29.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"refs":[{"id":"#212","type":"pr"}],"status":"done"},"kind":"result.reported","podId":"pod-checkout","prev":"1daff17c56410a04940183bfb8a19272ec9162aa5b53e8982a8badf6c51000f0","runId":"run-ana-0001","schema":"seats.record","seq":29,"taskId":"TASK-101","ts":"2026-10-25T16:00:30.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":2,"durationMs":412000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"4e47933b81d799b01bac98cb915f0b9d46092a62295f4f1dd62389282a3022e6","runId":"run-ana-0001","schema":"seats.record","seq":30,"taskId":"TASK-101","ts":"2026-10-25T16:00:31.000Z","v":1} diff --git a/packages/seats/docs/examples/run-ben-0001.jsonl b/packages/seats/docs/examples/run-ben-0001.jsonl new file mode 100644 index 0000000000..b6fd8ce3e9 --- /dev/null +++ b/packages/seats/docs/examples/run-ben-0001.jsonl @@ -0,0 +1,12 @@ +{"actor":{"id":"u-ben","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ben-review-1","issuedBy":"u-ben","scope":{"audience":["u-ben"],"connectionId":"conn-ben-github","tools":["github.get_pull_request"],"writeMode":"read"},"target":{"id":"u-agent-ben-goose","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":null,"runId":"run-ben-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:33.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENAI_API_KEY"]},"kind":"credential.granted","podId":"pod-checkout","prev":"ab5416e866a17b4fa875a4127be03bd14d2294d9a5f1271b84fb8dd4fa6abfff","runId":"run-ben-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:34.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"adapter":"goose","attempt":1,"cli":{"name":"goose","version":"1.54.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"6b2d6a9397ea15e894dc3f580acf90af4326073a8b3267ef869be8d04dcc8c97","runId":"run-ben-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:35.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","kind":"read","name":"github.get_pull_request","status":"pending","surface":"connector","title":"Read pull request #212","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"2e671132e0b51bea5f40b804e58badc66082619d6da280c9538f12ec8d0ae38a","runId":"run-ben-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:36.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","askedOf":{"id":"grant:grant-ben-review-1","role":"policy"},"grantId":"grant-ben-review-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"Read pull request #212","toolCallId":"tc-1"},"kind":"permission.requested","podId":"pod-checkout","prev":"647dfa8b21dba1911c740b2eb0ca1a2f38eade8adedf53ab54f44999701ae38c","runId":"run-ben-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:37.000Z","v":1} +{"actor":{"id":"grant:grant-ben-review-1","type":"policy"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ben-review-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"read"},"toolCallId":"tc-1"},"kind":"permission.decided","podId":"pod-checkout","prev":"36f32b99f8e1e72f616a5e1bccb4190b150e5d45301066f9fbd4a07342ff36f5","runId":"run-ben-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:38.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"1b5374fb6b7d6157ab2baa137b9294f1df5ee4d1018ff0ffbaff5cb54207d044","runId":"run-ben-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:39.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"text":"Reviewed #212: the rounding fix is right and the half-cent case is tested."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"75b890cf31a88f3e147662c84844f3c7054554ef2ad8cc3b7855ab93d7a18405","runId":"run-ben-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:40.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"6bcb6d2e835bebdd2bd2e124ee9f124518347642cebf6fc645c4e57e278120b9","runId":"run-ben-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:41.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"doneBy":"u-ben"},"kind":"task.done","podId":"pod-checkout","prev":"a24cfa42f1edfff156f029724d40e84ef51874ef3d311920c824023e42ff528e","runId":"run-ben-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:42.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"grantIds":["grant-ben-review-1"],"reason":"task.done"},"kind":"grant.revoked","podId":"pod-checkout","prev":"2b9a41ec20f94c8da8fe6bab7e0c90ba4002aa3d36fb167f30ca6286e3bc66b8","runId":"run-ben-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:43.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":1,"durationMs":95000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"0518c6a4515f9cac06b19169c84cd897c3dbc81757327c25e251cbf56f5a2ffa","runId":"run-ben-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:44.000Z","v":1} diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md new file mode 100644 index 0000000000..bc1a392764 --- /dev/null +++ b/packages/seats/docs/record-v1.md @@ -0,0 +1,658 @@ +# Run record, format v1 + +A run record is an append-only log of one agent run: +- who claimed which task; +- which access the run was granted, and by whom; +- which tool calls were asked about, and who decided them; +- what the agent attempted, and what came of it. + +The format is plain JSON lines, so a tool that has never seen this code can +read it, verify it and export it. + +One file holds one run. Several runs of the same task (a retry under a new +run, another person's agent reviewing the work) are several files that share +a `podId` and a `taskId`. + +The reference implementation: +- `src/record/format.js` writes and verifies records; +- `src/record/otel.js` exports them as OTLP; +- `src/trace/store.js` is the store that writes them. + +A generated sample is in [`examples/`](examples/) (section 7). + +## 1. Envelope + +Each line is one JSON object, followed by `\n` (UTF-8): + +```json +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":1,"cli":{"name":"opencode","version":"1.19.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"8de2801011b74ef1386e35eaa53bf6e0d5fce3692309ba832bc44a9005c2b97c","runId":"run-ana-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:05.000Z","v":1} +``` + +| field | type | meaning | +|---|---|---| +| `v` | integer | Format version. This document is `1`. | +| `schema` | string | Always `"seats.record"`. | +| `runId` | string | The run, `[A-Za-z0-9_-]{6,64}`. The same on every line of a file. | +| `taskId` | string, optional | The board's task id, verbatim (e.g. `"TASK-490"`). The same on every line when present. | +| `podId` | string; **required when `taskId` is set** | The pod the run belongs to. Task ids restart in each pod, so `taskId` is unique only together with `podId`. The same on every line when present. | +| `seq` | integer | 1 on the first line, then +1 per line, with no gaps. | +| `ts` | string | RFC 3339 timestamp in UTC: `YYYY-MM-DDTHH:MM:SS(.fraction)Z`. | +| `kind` | string | What happened (section 2). | +| `actor` | object or `null` | Who caused the entry (below). | +| `data` | object | Kind-specific fields (section 2). | +| `prev` | string or `null` | Lowercase hex SHA-256 of the previous line's bytes. `null` on `seq` 1. | + +**`actor`** is `{type, id, owner?, onBehalfOf?, name?}`: +- `type` is `agent`, `human`, `policy` or `system`. +- `id` is an identity, never a display name. + - For `agent` and `human` it is the instance's user id. Agents are users too. + - For `system` it names the writing component: `seats` (the run writer) + or `board`. + - For `policy` it names the grant whose rules decided (`grant:`), + or `policy`. + - It is a token (`[A-Za-z0-9][A-Za-z0-9._:-]{0,127}`), so a name with spaces + can't pass for one. +- `owner` is set on `agent` actors only. It is the user id of the person the + agent belongs to. +- `onBehalfOf` is set on `agent` actors only, and is optional. It is the + human whose request the agent is acting on (below). +- `name` is an optional display name. It is never part of the identity. + +The owner of an agent and the person who approves what it does are separate +fields on purpose: +- `actor.owner` says whose agent this is; +- the approver is `decidedBy` on `permission.decided`. + +In the case this format exists for, they differ: Ana's agent asks, and Ben +approves. + +**`actor.onBehalfOf`** is `{id, role}`. It records the pair an audit trail +needs for an agent's action: the agent, and the human it acted for +(ADR-025 D18). `owner` can't say this, because it only says whose agent it is. +- `id` is the human's user id, an id token under the same rule as `actor.id`. +- `role` is one of: + - `requester`: the human whose message started the run; + - `assigner`: the human who assigned the task. +- No other key is allowed. A writer throws on an unknown role or key, and a + verifier reports `bad_envelope` (`actor`) for a line that has one. +- It is allowed on `agent` actors only. Any other actor type carrying it is a + malformed actor. +- It lives on the actor, never in `data`, so it covers every entry the agent + authors (`tool.call` and `permission.requested` included), and the pair is + never split across two places. +- A run with no human behind it (a schedule, a heartbeat, an agent-to-agent + DM) writes no `onBehalfOf`. A writer never fills it in with the agent's + owner: `owner` already says whose agent it is, and `onBehalfOf` means a + human asked. The writer does not enforce this by comparing `id` to `owner`, + because the owner can really be the requester (Ana asks her own agent), and + that line must be accepted. + +Adding it kept `v: 1` (section 3, rule 3), and a line without it is +byte-identical to one written before the field existed. Ana's agent, acting +on a message from Ben, calls a Sentry tool: + +```json +{"actor":{"id":"u-agent-ana-opencode","onBehalfOf":{"id":"u-ben","role":"requester"},"owner":"u-ana","type":"agent"},"data":{"name":"sentry.resolve_issue","surface":"connector","title":"Resolve CHECKOUT-88","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"cbe33a93d273b1ad959fb6e836995ac8c9a119921ddbcce7b785c7d63ccf2a9e","runId":"run-ana-0002","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:01:05.000Z","v":1} +``` + +`actor: null` means the writer did not know who caused the entry. A reader +must not fill it in. Each kind has a default actor (section 2), which a +writer records when the emitter names none. + +**Canonical serialization.** A line is the JSON Canonicalization Scheme +(RFC 8785) form of its entry: +- object keys are sorted by UTF-16 code units; +- there is no insignificant whitespace; +- strings and numbers are serialized as ECMAScript `JSON.stringify` does; +- an absent field is omitted, never written as `null`. `actor` and `prev` are + the two fields where `null` is a value. + +A writer in any language that follows these rules produces byte-identical +lines. A verifier rejects a line that is not in canonical form. + +**What `prev` covers.** `prev` is SHA-256 over the exact bytes of the previous +line, without its trailing `\n`. It is not taken over a re-serialization, so +a verifier needs no JSON library to check the chain. + +## 2. Kinds + +Every kind below is part of v1. "Default actor" is the actor a writer records +when the emitter names none. "run" means the run's own actor, usually the +agent. + +### Claim + +The board is the source of truth for tasks. Its states are `pending`, +`claimed`, `done` and `blocked`. The record only observes the board: these +entries mirror what the board already holds, and they never change it. + +| kind | data | default actor | +|---|---|---| +| `task.claimed` | `{leaseUntil?}` | run | +| `task.released` | `{reason?}`, such as `blocked`, `lease_expired`, `reassigned` or `abandoned` | run | +| `task.done` | `{doneBy?}` (a user id) | `{type: system, id: board}`. The board is its writer. | + +`task.done` needs a run with a `taskId`. Writing it ends every open grant +bound to that task (below). + +### Grant + +A grant has the shape of a room grant: a connection, exact tool names, a +write mode, a budget and an audience, plus its parent for attenuation. + +| kind | data | default actor | +|---|---|---| +| `grant.issued` | `{grantId, parentGrantId?, scope, target?, issuedBy?, expiresAt?, binding?}` | derived from `issuedBy`: `policy` gives a policy actor, a user id gives a human actor | +| `grant.revoked` | `{grantIds: [grantId, …], reason}` | `system` | + +**`scope`** is `{connectionId?, tools: [name], writeMode?, budget?, audience: [userId]}`: +- `connectionId` is the connection whose credential the grant uses. The + credential itself is never named or recorded. A broker resolves it. +- `tools` are exact tool names, such as `sentry.resolve_issue`, never patterns. +- `writeMode` is `read`, `write` or `write-with-confirm`. +- `budget` is `{calls?, windowMs?}`. +- `audience` lists the user ids the grant was minted for. + +**The other fields:** +- `parentGrantId` is the grant this one attenuates, if any. Following it gives + the lineage. +- `target` is `{kind: pod|seat, id}`: whom the grant is to. +- `issuedBy` is a user id, or `policy`. +- `binding` is `{taskId, podId, endsOn}`, and it must name the run's own + `taskId` and `podId`. `endsOn` is one of: + - `task.done`: the grant ends when the board marks its task done; + - `expiresAt`: it ends at `expiresAt`, which is then required. + +**`grant.revoked`** ends every id in `grantIds` in a single entry. `reason` is +one of: +- `task.done`: the bound task is done. The writer records this right after `task.done`; +- `expired`: `expiresAt` passed; +- `cascade`: an ancestor was revoked. `grantIds[0]` is that ancestor, and the + rest are its descendants, all ended by the same write; +- `manual`: a person or policy revoked exactly these grants. The actor says who. + +The reasons are unchanged by the 2026-10-25 revision. In a hosted pod, server +writers (TASK-513) supply `expired` and `cascade`, because the server holds +grant expiry and lineage. The local store writes the same two reasons from +`expireGrants` and `revokeGrant` for a run that has no server writer. + +**What revocation means in v1.** A revoked grant is no longer valid. A +credential that a run placed in its agent process's environment stays there +until that process exits. A revocation records that the grant's term ended; +it does not prove the process lost access. That happens at `run.settled`. + +### Approval + +| kind | data | default actor | +|---|---|---| +| `permission.requested` | `{toolCallId?, options: [{optionId, kind}], question, askedOf: {id, role}, grantId?, decisionCardId?, argsDigest?}` | run | +| `permission.decided` | `{outcome, optionId?, kind?, toolCallId, rule, approval?, decidedBy, decidedByRole, decisionCardId?, grantId?, argsDigest?}` | the decider: a human actor for a user id, a policy actor for `policy` | +| `decision.ruled` | `{decisionCardId, value, decidedBy, decidedByRole}` | the decider, as for `permission.decided` | + +**The request:** +- `question` is required: the human-readable ask, normally the tool call's + title. It is agent text, so it is scrubbed of known credential values + before it is written, like other agent text (section 3, rule 1). +- `askedOf` is required: `{id, role}`. + - `id` is an id token, under the same rule as `actor.id`. + - `role` is from the `decidedByRole` enum below. + - A call that a rule decides, with no person asked, is asked of the policy: + `{id: "grant:", role: "policy"}`. +- `grantId` is the grant the call would act under. +- `decisionCardId` is the card the ask rides on, when there is one. +- `argsDigest` identifies the call's arguments (under Tool activity, below). + +**The decision:** +- `outcome` is `selected` or `cancelled`. +- `kind` is the ACP option kind chosen: `allow_once` approves, and + `reject_once` refuses. +- `rule` is `{list, pattern}`: + - `list` is `allow`, `ask`, `deny` or `default` (no rule matched); + - `pattern` is the run policy's pattern that matched, or `null`. +- `approval` is present only when someone was asked. It is `approved`, + `rejected`, `timed_out`, `failed` or `aborted`. + +**Who decided:** +- `decidedBy` is required: a user id, or `"policy"`. A writer refuses `null`. + - A rule-decided call is `"policy"`. So is the default given when no + answer came in time, or when the answer did not name a user and an owner + role: such an answer approves nothing. + - A permission hook's answer is held to the same rule. An answer that does + not name who decided (`decidedBy` a user id, plus an owner + `decidedByRole`) gets the reject default, recorded as + `decidedBy`/`decidedByRole` `policy`/`policy` with `approval: failed`. A + decider marked `recordsPermissions` (the valet's policy) writes and + attributes its own entries, so it is exempt. + - It is `"policy"` exactly when `decidedByRole` is `"policy"`. +- `decidedByRole` is one of: + - `credential-owner`: the owner of the connection the call would use. This + is who decides approvals today; + - `decision-owner`: a person assigned to the decision. This is reserved, + and writers may not emit it yet; + - `agent-owner`: the owner of the asking agent; + - `pod-member`: a person in the pod who owns neither the credential nor + the asking agent. It is valid only on `decision.ruled`. A parked call is + decided by an owner or by policy, so `permission.decided` and + `permission.requested.askedOf` refuse it; + - `policy`. + + It is required on `permission.decided`, and a writer refuses `null`. +- `decisionCardId` is set when the decision came through a decision card. +- `grantId` ties the decision to its `grant.issued`. + +**`decision.ruled`** records a ruling on a decision card that is not tied to +a parked tool call, such as whether to retry a stuck attempt. +- `decisionCardId` is required. +- `value` is the ruling: a string, number or boolean. +- `decidedBy` is required. It is a user id, or `"policy"`. +- `decidedByRole` is required, and comes from the same enum. Only here may it + be `pod-member`. + +A ruling on a card that a `permission.requested` carried belongs with that +request, and the OTLP export places it there. + +### Attempt + +| kind | data | default actor | +|---|---|---| +| `attempt.started` | `{attempt, adapter, cli?: {name, version}, sessionId?}` | run | +| `session.opened` | `{method, sessionId, requestedSessionId?, fallback?}` | `system` | +| `attempt.settled` | `{attempt, outcome, failure?: {kind, detail}}` | `system` | +| `run.settled` | `{outcome, failure?, durationMs, attempts}` | `system` | + +- `outcome` is `ok`, `failed`, `stuck` or `cancelled`. +- `failure.kind` is a key of `FAILURE_KINDS` in `src/failures.js`, or `null` + when the cause had no name. +- The agent's id and owner are the actor of `attempt.started`. +- `session.opened` records the agent session an attempt runs in, as an ACP + adapter opens it: + - `method` is `new`, `resume` or `load`; + - `sessionId` is the session the attempt runs in, a non-empty string; + - `requestedSessionId` and `fallback` are set together, only when + `method` is `new`: a session was asked for and could not be reopened. + `fallback` is one of two forms, `agent supports neither session/resume + nor session/load`, or `session/ answered error ` + with the JSON-RPC error's integer code; + - no other field is allowed. +- `run.settled` is the last line of a complete record. Nothing is written + after it. +- If an attempt has no `attempt.settled`, the record says only that the next + `attempt.started`, or `run.settled`, came after it. A reader must not infer + the outcome. + +### Tool activity + +| kind | data | default actor | +|---|---|---| +| `tool.call` | `{toolCallId, title, kind?, status?, name?, surface?, argsDigest?}` | run | +| `tool.update` | `{toolCallId, status?, title?}` | run | + +- `kind` is the ACP tool kind (`read`, `edit`, `execute`, `fetch`, …). +- `name` is the tool's own name, when known. +- `surface` says where the call went: + - `connector`: a third-party service, through a connector; + - `commonly`: the room's own tools (`commonly_*`); + - `native`: the CLI's built-in tools. + + All three land in the same record. +- `argsDigest` is a digest of the call's arguments, never the arguments: + `"sha256:"` followed by the lowercase hex SHA-256 of the arguments' + RFC 8785 canonical form, the same serialization as a line. Key order does + not change it. For example, `{"b":2,"a":1}` is digested as the bytes + `{"a":1,"b":2}`. + - A writer refuses any other value, so raw arguments can't ride in under + this name. + - The same arguments give the same digest on `tool.call`, + `permission.requested` and `permission.decided`, so a reader can match a + call to the approval it ran under. + - The valet writes it on both permission entries when the ACP tool call + carries `rawInput`. An adapter writes it on `tool.call` with + `argsDigestOf(rawInput)`. + - A digest of low-entropy arguments can be confirmed by guessing them. It + hides the values from a casual reader, not from someone who can guess them. + +**Conformance gate for writers in other languages.** Every record writer, +in any language, must pass every vector in +[`test/vectors/args-digest.json`](../test/vectors/args-digest.json), and a +writer that fails it does not ship. `test/args-digest-vectors.test.js` holds +the file to `argsDigestOf`, so the file is the definition, not a sample. The +vectors cover: +- key order by UTF-16 code units; +- string escaping, with U+2028 and U+2029 written raw; +- number formatting: `1.0` is `1`, `-0` is `0`, `1e21`, small floats, and an + integer past 2^53, which a writer parses as an IEEE 754 double first, so + `9007199254740993` on the wire is written `9007199254740992`; +- the cases where a hand-rolled key-sorted serializer drifts: an undefined + value in an object drops its key, one in an array becomes `null`, and `NaN` + or `Infinity` anywhere means the arguments have no digest. + +A lone surrogate (half of a UTF-16 pair) in any string or key also means no +digest, as RFC 8785 requires. Only the digest refuses it: a line still +records agent text holding one, as a `\u` escape, because a stream cut +mid-pair must not lose the chunk. On arguments that can't be digested, a +writer omits `argsDigest` rather than writing a different digest, because a +different digest would not match the approval the call ran under. A writer +built in Go runs its tests with `-race` in CI. + +### Credentials + +| kind | data | default actor | +|---|---|---| +| `credential.granted` | `{names: [NAME]}` | `system` | +| `credential.denied` | `{name, reason}`, where `reason` is `not_granted`, `unresolved` or `too_short` (a value under 8 characters) | `system` | + +These cover the environment credentials a run is started with. Values never +appear here: `NAME` is an environment variable name matching +`[A-Z_][A-Z0-9_]*`. + +### Sandbox + +| kind | data | default actor | +|---|---|---| +| `sandbox.applied` | `{sandbox, reason?}` | `system` | + +- `sandbox` is the OS sandbox the run's process was started in: `seatbelt` + (macOS), `bwrap` (Linux), or `none`. +- `reason` says why there was none. It is set only when `sandbox` is `none`, + and is a non-empty string. A writer may leave it out when it has no + reason to give. +- No other field is allowed. +- It is its own group, not part of Attempt: it records how the process was + confined, not what the attempt did or how it ended. + +### Egress + +A sandboxed run reaches the network only through the egress proxy, which +admits the hosts its grant names, plus its model provider. These lines +record what the proxy did. + +| kind | data | default actor | +|---|---|---| +| `egress.allowed` | `{host, port}`: a tunnel opened | `system` | +| `egress.denied` | `{host, port, reason}`: a request was refused | `system` | + +- `host` is a lowercase DNS name, an IP literal (IPv6 without brackets), or + the literal string `"[redacted]"` when the requested name held one of the + run's credentials. +- `port` is an integer from 1 to 65535. +- `reason` is one of: + - `bad_request`: the request could not be parsed; + - `not_connect`: a method other than `CONNECT`; + - `ip_literal`: the target was an address, not a name; + - `not_allowed`: no grant entry names the host and port; + - `unresolved`: the name did not resolve; + - `private_address`: it resolved to a non-public address that no exact + entry names; + - `unreachable`: the connection failed. +- `host` and `port` are `null` together, and only on a malformed request + (`bad_request`, or `not_connect` when the target can't be read). Every + other line names both. +- No other field is allowed, so no header, body or resolved address is + recorded. + +### Result + +| kind | data | default actor | +|---|---|---| +| `result.reported` | `{status, refs?: [{type, id}]}` | run | + +- `status` is `done`, `partial`, `blocked` or `failed`. +- A ref points at what the run produced, such as `{type: "pr", id: "#212"}`. + +### Run activity + +These kinds carry the trace vocabulary through unchanged. Their data is +documented in `src/trace/events.js`. + +| kind | default actor | +|---|---| +| `turn.prompt_sent` | `system` | +| `agent.message_chunk`, `agent.thought_chunk`, `agent.retry`, `plan`, `usage` | run | +| `liveness.stalled`, `liveness.recovery`, `process.exited` | `system` | + +`usage` may also carry `contextTokens` (how much of the agent's context +window is in use) and `contextSize` (the window's size). Each is an integer +`>= 0`, or `null` when unknown; a writer refuses any other value. + +`agent.retry` is the agent reporting that it is retrying its provider: +`{state, attempt?, nextRetryAt?, status?, errorType?}`. +- `state` is `scheduled` (a retry waits until `nextRetryAt`) or `started` + (the waited-for attempt went out). +- `attempt` is an integer `>= 1`, and `nextRetryAt` an RFC 3339 UTC + timestamp. +- `status` is an integer HTTP status, `100` to `599`. +- `errorType` is one of `provider.auth`, `provider.content-filter`, + `provider.free-limit`, `provider.internal`, `provider.invalid-output`, + `provider.invalid-request`, `provider.no-route`, `provider.quota`, + `provider.rate-limit`, `provider.timeout`, `provider.transport`, + `provider.unknown`, `provider.unsupported-operation` or `other`. These are + the provider error types opencode 2.1.4 reports on a retry; a writer maps + any other type to `other`. +- A field that is not known is absent, never `null`. The agent's error + message and the provider's response body are never recorded, and a writer + refuses any field not listed here. + +### Mapping from the trace vocabulary + +Emitters keep calling `emit(type, data)` with `EVENT_TYPES` names, and the +store writes the record kind: + +| `EVENT_TYPES` entry | record kind | +|---|---| +| `run.started` | `attempt.started` (renamed: it is emitted once per attempt) | +| every other entry | the same name | + +What's new or changed in `EVENT_TYPES`: +- `task.claimed`, `task.released`, `task.done`, `grant.issued`, + `grant.revoked`, `attempt.settled`, `decision.ruled` and + `result.reported` are new entries. +- `emit` also accepts the record kind `attempt.started`. +- `readRun` returns each entry with its trace name as `type` (`run.started` + for an `attempt.started` line), and puts the envelope fields beside it. + +## 3. Rules + +1. **No secret values, by construction.** + - No field of any kind holds a credential value. A grant names a + connection id. A run's environment credentials appear only as names, + and a writer refuses one that is not name-shaped. + - Agent-authored text, such as message chunks and tool titles, belongs to + the agent and could repeat a value the agent holds. A writer that knows + the run's credential values replaces every occurrence of each, whatever + its length, with `[redacted]` before the line is hashed. A writer does + not grant a credential shorter than 8 characters (`credential.denied`, + `too_short`). + - Message and thought chunks are scrubbed per stream, not per line, so a + value split across chunks is caught. A writer holds back a chunk's tail + while it could begin a known value, and writes it with the stream's next + chunk. If the stream ends first, the held tail is a partial value, so the + writer writes `[redacted]` as a chunk of its own in its place, before the + next `attempt.started`, `attempt.settled` or `run.settled`. A short + ordinary tail that happened to begin a value is redacted too. A held tail + lives only in the writer's memory: a writer that crashes before the next + write loses it, and never writes it. + - That replacement covers known values only. A value the writer was not + given, or one the agent transformed (base64, say), passes through. +2. **Readers ignore unknown fields and skip unknown kinds.** A reader that + meets a kind it doesn't know still checks the chain over that line. +3. **`v` bumps only on a breaking change:** + - a field removed or renamed; + - a meaning changed; + - a change to the hash or serialization rules. + + Adding a kind, an optional field or an enum value doesn't break anything, + and keeps `v: 1`. A reader refuses a `v` higher than it knows. + + The 2026-10-25 revision kept `v: 1`. It added `decision.ruled`, + `argsDigest`, and `grantId` and `decisionCardId` on + `permission.requested`. It also made `question` and `askedOf` required + for writers of `permission.requested`. + - Readers must not assume those two fields on lines written before the + revision. + - The verifier checks the envelope and the chain, never data, so those + lines still verify. + + A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`, + the kinds `session.opened`, `sandbox.applied`, `egress.allowed`, + `egress.denied` and `agent.retry`, `contextTokens` and `contextSize` on + `usage`, and the `pod-member` value of `decidedByRole`. Lines without them + are unchanged. +4. **One writer per file.** A single process appends lines in `seq` order. + +## 4. What the hash chain proves, and what it doesn't + +**It detects:** +- an edited byte in any line except the last; +- a deleted line, which leaves a `seq` gap and a `prev` mismatch; +- lines reordered within the file; +- a torn final line, where a crash stopped a write partway. + +**The last line** is covered only against an outside record of the head: +- When a run settles, the store writes `head` (the SHA-256 of the last line) + and `entries` to its index, and `verifyRun` checks both. +- The index sits in the same directory and is just as editable, so this + catches accidents, not an adversary. +- A record cut short before `run.settled` is a valid prefix. It is reported + as unsettled, not as tampered. + +**It does not prove:** +- that any event happened. The writer could have recorded anything; +- who wrote the file. There is no signature in v1; +- that the whole file wasn't replaced by a different chain that is consistent + with itself; +- anything across files. Two runs of one task are not chained together. + +So the chain shows that the file is the one its writer wrote, in the order +it was written. Whether the writer told the truth is a question for whoever +operates it. + +## 5. OpenTelemetry mapping + +`toOtlpJson(entries, {serviceName})` returns an OTLP/JSON +`ExportTraceServiceRequest`, nested as `resourceSpans` → `scopeSpans` → +`spans`. The resource carries `service.name`, and the scope is +`seats.record`, version `1`. + +### Shape + +- **A task is a trace.** `traceId` is the first 16 bytes of + SHA-256(`"seats.record/trace/task/" + podId + "/" + taskId`). A run with no + task uses SHA-256(`"seats.record/trace/run/" + runId`). Every run of one + task lands in one trace. +- **A run is a span:** `seats.run`, kind INTERNAL, the root of its run. +- **An attempt is a child span of its run:** `invoke_agent {agent name}`. +- **A tool call is a child span of its attempt:** `execute_tool {tool name}`. +- **Decisions are span events.** A `permission.*` entry goes on the tool span + with the same `toolCallId`. Without one, it goes on the current attempt, + then the run. A `decision.ruled` entry goes on the span of the request + that carried its `decisionCardId`, and otherwise on the span current at + the time. +- **Every other kind is an event on the span current at the time.** That + covers grants, credentials, claims, `task.done`, results, liveness, + `session.opened`, `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. +- **Span ids** are the first 8 bytes of + SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to + the entry that opens the span. The run span uses `0`. Exporting the same + entries always produces the same bytes. +- **Times** come from the entries' `ts`: + - A span ends at its closing entry: `attempt.settled`, the next + `attempt.started`, `run.settled`, or a tool's `completed` or `failed` + update. + - A span with no closing entry ends when its parent does. +- **Status:** + - The outcomes `failed`, `stuck` and `cancelled`, and a tool update with + status `failed`, set `STATUS_CODE_ERROR`. + - The error class goes in `error.type`: the failure kind, or `tool_failed`. + - Success leaves the status unset, as OpenTelemetry recommends for + instrumentation. +- **Not exported:** message and thought chunks are left out, following the + upstream conventions, which make content capture opt-in. Unknown kinds are + skipped. + +### OpenTelemetry GenAI conventions used + +The source is github.com/open-telemetry/semantic-conventions-genai, read on +2026-10-25 at commit `9ffc7e00`. The GenAI pages on opentelemetry.io now say +they have moved there. + +**Upstream status:** both documents used here (`gen-ai-agent-spans.md` and +`gen-ai-spans.md`), and every `gen_ai.*` attribute below, are **Development** +status. They can change without a major version, so this mapping is pinned +to that commit. `error.type` is Stable. + +| attribute | on | from | +|---|---|---| +| `gen_ai.operation.name` | attempt, tool | `invoke_agent`, `execute_tool` | +| `gen_ai.agent.id` | attempt | `actor.id` of `attempt.started` | +| `gen_ai.agent.name` | attempt, tool | `actor.name`, else `actor.id` | +| `gen_ai.conversation.id` | attempt | `sessionId` of `attempt.started`, else of the attempt's `session.opened` | +| `gen_ai.tool.name` | tool | `name`, else `title` | +| `gen_ai.tool.call.id` | tool | `toolCallId` | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | attempt | the last `usage` entry in the attempt | +| `error.type` | any span with error status | failure kind, or `tool_failed` | + +### Choices and gaps + +- **The attempt span is INTERNAL, not CLIENT.** The agent runs in a child + process, so neither upstream variant fits exactly. The CLIENT variant of + `invoke_agent` makes `gen_ai.provider.name` Required, and a supervisor + cannot know which provider a CLI chose. +- **`gen_ai.tool.type` (Recommended) is not set.** Its upstream values + (`function`, `extension`, `datastore`) don't map cleanly onto `surface`. +- **Everything else uses the neutral `seats.*` namespace:** + - these named attributes: `seats.run.id`, `seats.task.id`, `seats.pod.id`, + `seats.run.outcome`, `seats.run.attempts`, `seats.attempt.number`, + `seats.attempt.outcome`, `seats.adapter`, `seats.cli.name`, + `seats.usage.context_tokens`, `seats.usage.context_size` (the last + known values in the attempt), + `seats.cli.version`, `seats.agent.owner`, `seats.agent.on_behalf_of.id`, + `seats.agent.on_behalf_of.role`, `seats.tool.kind`, + `seats.tool.surface` and `seats.tool.args_digest`; + - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner`, + `.name`, `.on_behalf_of.id` and `.on_behalf_of.role`; + - on the tool span, `seats.actor.on_behalf_of.id` and `.role` from the + `tool.call` actor, so the pair sits on the action itself; + - the event's data, flattened as `seats.` with camelCase segments in + snake_case. For example, `decidedBy` becomes `seats.decided_by`, and + `scope.connectionId` becomes `seats.scope.connection_id`, and + `askedOf.id` becomes `seats.asked_of.id`. + +## 6. Not in v1 (open for v2) + +- **Signing.** The writer signs each line, or each head, so that "who wrote + this" can be proved and a replaced file can be detected. +- **Multi-writer chains.** A pod-level record that several hosts append to + needs ordering across writers: one chain per writer plus a merge, or a + sequencer. +- **Cross-run linking.** A run's first line could be chained to the previous + run of the same task, or each task could keep a head, so that a deleted run + can be detected. This also closes a v1 gap. When a task is marked done after + a run that held a task-bound grant has already settled, that run's file has + no `grant.revoked` line. The sample shows this with Ana's grant. +- **Live revocation.** A brokered credential, behind a proxy rather than in an + environment variable, would let `grant.revoked` take effect inside a running + process. +- **Pod-level views.** These are a filter on `podId`, which v1 already writes + on every task-bound line. + +## 7. Sample + +`examples/` is generated by `test/record-example.test.js`. A test fails if the +files drift from what the code writes. + +- `run-ana-0001.jsonl`: Ana's agent (opencode 1.19.0) claims `TASK-101` in + `pod-checkout`. + - Its grant attenuates Ben's Sentry connection grant. + - The first attempt is stuck. Ana rules `retry` on card `card-2b9` as + `agent-owner` (`decision.ruled`), and the agent is retried. + - A `git push` is denied by the policy. + - The headline case: Ana's agent asks a question about a Sentry write. The + `permission.requested` line is tied to Ana's grant and to card + `card-7f3`, and is asked of Ben as `credential-owner`. Ben approves it on + that card. `tool.call` and both permission entries carry the same + `argsDigest`. + - The run reports a PR. +- `run-ben-0001.jsonl`: Ben's agent (goose 1.54.0) reviews the PR under a + read grant bound to the task. The board then marks the task done, which + revokes that grant in the same run. +- `TASK-101.otlp.json`: both runs exported as one trace. diff --git a/packages/seats/package.json b/packages/seats/package.json new file mode 100644 index 0000000000..ca4b81f327 --- /dev/null +++ b/packages/seats/package.json @@ -0,0 +1,22 @@ +{ + "name": "@commonlyai/seats", + "version": "0.0.0", + "private": true, + "license": "Apache-2.0", + "description": "Adapters that run agent CLIs as supervised seats. Bundled inside @commonlyai/cli; not published on its own.", + "type": "module", + "exports": { + ".": "./src/index.js", + "./*": "./src/*.js" + }, + "files": [ + "src", + "README.md" + ], + "engines": { + "node": ">=20.0.0" + }, + "scripts": { + "test": "node --test \"test/**/*.test.js\"" + } +} diff --git a/packages/seats/src/adapters/hermes.js b/packages/seats/src/adapters/hermes.js new file mode 100644 index 0000000000..e54973f88f --- /dev/null +++ b/packages/seats/src/adapters/hermes.js @@ -0,0 +1,206 @@ +/** Hermes 0.22.2 local CLI wrapper. Hermes owns provider I/O; this file only spawns it. */ +import { spawn as childSpawn, spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import { mkdir } from 'fs/promises'; +import { homedir } from 'os'; +import { join } from 'path'; +import { StringDecoder } from 'string_decoder'; +import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { requireSeatModel } from '../seat-model.js'; +import { adapterFailure, spawnCredentials } from '../upstream-refusal.js'; + +export const TESTED_HERMES_VERSION = '0.22.2'; +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; +const VERSION_TIMEOUT_MS = 15000; +const MAX_STDERR_CHARS = 4096; +const MAX_LINE_CHARS = 1024 * 1024; + +const readVersion = (env, spawnSyncImpl = spawnSync) => { + let probe; + for (let attempt = 0; attempt < 2; attempt += 1) { + probe = spawnSyncImpl('hermes', ['--version'], { + encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS, + }); + if (probe.error?.code === 'ENOENT') return null; + if (probe.error || probe.status !== 0) continue; + const version = String(probe.stdout || '').match(/Hermes Agent v([^\s]+)/i)?.[1]; + if (version !== TESTED_HERMES_VERSION) { + throw new Error(`Hermes ${version || 'unknown'} is unsupported; this adapter requires ${TESTED_HERMES_VERSION}`); + } + return version; + } + const status = probe.status ?? 'none'; + const signal = probe.signal ?? 'none'; + const code = probe.error?.code ?? 'none'; + throw new Error(`Hermes version probe failed after 2 attempts (status=${status}, signal=${signal}, code=${code}, timeout=${VERSION_TIMEOUT_MS}ms); check the Hermes installation or startup log`); +}; + +export const readHermesJsonl = (line, state) => { + let event; + try { event = JSON.parse(line); } catch { throw new Error('Hermes emitted invalid JSONL'); } + if (state.result) throw new Error('Hermes emitted data after its terminal result'); + if (event?.type === 'system' && event.subtype === 'init') { + state.started = true; + } else if (event?.type === 'result') { + state.result = event; + } else if (!state.started) { + throw new Error('Hermes emitted an event before init'); + } +}; + +const runHermes = ({ args, prompt, cwd, env, timeoutMs, spawnImpl, credentials, onSpawn, controls = {} }) => new Promise((resolve, reject) => { + const child = (spawnImpl || childSpawn)('hermes', args, { + cwd, env, stdio: ['pipe', 'pipe', 'pipe'], + }); + observeSpawn(child, onSpawn, controls); + const state = { started: false, result: null }; + const decoder = new StringDecoder('utf8'); + let buffer = ''; + let stderr = ''; + let timedOut = false; + let settled = false; + let closed = false; + let stopping = false; + let killTimer; + const finish = (error, result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); else resolve(result); + }; + const stop = () => { + if (stopping || closed) return; + stopping = true; + child.kill('SIGTERM'); + killTimer = setTimeout(() => { + if (!closed) child.kill('SIGKILL'); + }, 5000); + }; + const failAndStop = (error) => { + stop(); + finish(error); + }; + const timer = setTimeout(() => { + timedOut = true; + stop(); + }, timeoutMs); + child.on('error', (error) => failAndStop(adapterFailure('Hermes', error?.message || 'spawn failed', { credentials }))); + child.stdout.on('data', (chunk) => { + if (settled) return; + buffer += decoder.write(chunk); + let newline = buffer.indexOf('\n'); + while (newline !== -1) { + if (newline > MAX_LINE_CHARS) { + failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); + return; + } + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) { + try { readHermesJsonl(line, state); } catch (error) { + failAndStop(error); + return; + } + } + newline = buffer.indexOf('\n'); + } + if (buffer.length > MAX_LINE_CHARS) { + failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); + } + }); + child.stdout.on('error', (error) => failAndStop(adapterFailure('Hermes stdout', error?.message || 'read failed', { credentials }))); + child.stderr.on('data', (chunk) => { + stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_CHARS); + }); + child.on('close', (code) => { + closed = true; + clearTimeout(killTimer); + if (settled) return; + if (timedOut) return finish(new Error(`Hermes timed out after ${timeoutMs}ms`)); + buffer += decoder.end(); + if (buffer.trim()) { + try { readHermesJsonl(buffer.trim(), state); } catch (error) { return finish(error); } + } + const result = state.result; + if (!result || !state.started) return finish(new Error('Hermes exited without a complete JSONL result')); + if (code !== 0 || result.exit_code !== 0) { + return finish(adapterFailure('Hermes', result.error || stderr || `exit ${code}`, { + credentials, exitCode: code, + })); + } + if (typeof result.text !== 'string' || !result.session_id) { + return finish(new Error('Hermes result omitted text or session_id')); + } + return finish(null, { text: result.text, newSessionId: result.session_id }); + }); + child.stdin.on('error', (error) => failAndStop(adapterFailure('Hermes stdin', error?.message || 'write failed', { credentials }))); + child.stdin.end(prompt); +}); + +/** + * @param {import('../contract.js').SeatHost & { profileRoot?: string }} [host] + * profileRoot is where each seat's isolated HERMES_HOME is created. + */ +export const createHermesAdapter = ({ + memoryPreamble = (prompt) => prompt, + effectiveTrust = (trust) => trust, + seatModelRequired = () => false, + profileRoot = join(homedir(), '.seats', 'hermes', 'profiles'), +} = {}) => ({ + get requiresModel() { return seatModelRequired(); }, + name: 'hermes', + runtimeType: 'hermes', + supportsProvider: false, + providerKeyEnv: 'OPENROUTER_API_KEY', + + async validateEnvironment(environment) { + if (environment?.provider !== undefined) { + throw new Error('Hermes adapter does not consume environment.provider; supply OPENROUTER_API_KEY in the host environment'); + } + if (environment?.mcp?.length) { + throw new Error('Hermes adapter does not yet consume environment.mcp declarations'); + } + if (effectiveTrust(environment?.sandbox?.trust) === 'public' + || (environment?.sandbox?.mode && environment.sandbox.mode !== 'none')) { + throw new Error('Hermes public or sandbox-declared seats are unsupported: no enforced confinement adapter exists'); + } + }, + + async detect() { + const version = readVersion(process.env); + return version ? { path: 'hermes', version } : null; + }, + + async spawn(prompt, ctx = {}) { + const seatModel = requireSeatModel(ctx.environment, 'hermes', seatModelRequired()); + await this.validateEnvironment(ctx.environment); + // Keep Hermes away from the operator's default profile, including any + // Nous Portal/Cloud credentials there. The wrapper provides the sole key. + const seat = `${ctx.agentName || ''}\0${ctx.cwd || ''}`; + const home = ctx._hermesHome || join(profileRoot, + createHash('sha256').update(seat).digest('hex').slice(0, 24)); + await mkdir(home, { recursive: true, mode: 0o700 }); + const env = { ...(ctx.env || process.env), HERMES_HOME: home }; + if (!readVersion(env, ctx._spawnSyncImpl)) { + throw new Error('Hermes Agent 0.22.2 is not on PATH'); + } + if (!env.OPENROUTER_API_KEY) { + throw new Error('Supply the operator OPENROUTER_API_KEY through Secret Manager before running Hermes'); + } + const args = ['chat', '--oneshot', '--query-file', '-', '--format', 'stream-json', + '--provider', 'openrouter', '--source', 'tool', '--no-restore-cwd']; + if (seatModel) args.push('--model', seatModel); + if (ctx.sessionId) args.push('--resume', ctx.sessionId); + const fullPrompt = memoryPreamble(prompt, ctx.memoryLongTerm, { + freshSession: !ctx.sessionId, + }); + const requestedTimeout = Number(ctx.timeoutMs); + const timeoutMs = Number.isFinite(requestedTimeout) && requestedTimeout > 0 + ? Math.min(requestedTimeout, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS; + return runHermes({ + args, prompt: fullPrompt, cwd: ctx.cwd, env, timeoutMs, + spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args, { modelFlag: '--model', valueFlags: ['--query-file', '--format', '--provider', '--source', '--resume'] }), + credentials: spawnCredentials(ctx, ['OPENROUTER_API_KEY']), + }); + }, +}); diff --git a/packages/seats/src/adapters/stub.js b/packages/seats/src/adapters/stub.js new file mode 100644 index 0000000000..9cae043466 --- /dev/null +++ b/packages/seats/src/adapters/stub.js @@ -0,0 +1,24 @@ +/** + * Stub adapter — the no-op reference implementation of the ADR-005 adapter + * contract. Used by `commonly agent attach stub` and the Phase-1a test + * harness so the run loop can be reviewed end-to-end without a real CLI + * on PATH. + * + * Contract (ADR-005 §Adapter pattern): + * detect(): Promise<{ path, version } | null> + * spawn(prompt, ctx): Promise<{ text, newSessionId?, memorySummary? }> + * + * The adapter MUST be pure: input (argv + env + prompt) → output (text + optional + * next session-id + optional memory summary). No direct network, no direct API + * calls, no hidden state. The run loop handles all Commonly-facing I/O. + */ + +export default { + name: 'stub', + async detect() { + return { path: '(builtin)', version: '0.0.0' }; + }, + async spawn(prompt, _ctx) { + return { text: `(stub) received: ${String(prompt ?? '').slice(0, 200)}` }; + }, +}; diff --git a/packages/seats/src/contract.js b/packages/seats/src/contract.js new file mode 100644 index 0000000000..3980951746 --- /dev/null +++ b/packages/seats/src/contract.js @@ -0,0 +1,48 @@ +/** + * The adapter contract (ADR-005 §Adapter pattern). An adapter turns one prompt + * into one CLI run and one reply. It owns argv, env and output parsing for its + * CLI; the caller owns everything that talks to a Commonly instance. + * + * @typedef {object} Adapter + * @property {string} name + * @property {string} [runtimeType] + * @property {string} [providerKeyEnv] host env var the adapter's provider key arrives in + * @property {() => Promise<{ path: string, version: string } | null>} detect + * @property {(environment: object) => Promise} [validateEnvironment] refuse before spawn + * @property {(prompt: string, ctx: object) => Promise} spawn + * + * @typedef {object} SpawnResult + * @property {string} text + * @property {string} [newSessionId] + * @property {number} [contextTokens] + * @property {string} [memorySummary] + * + * Adapters that need host policy are built by a factory that takes a SeatHost. + * Every field is optional; an absent field takes the neutral default named. + * + * @typedef {object} SeatHost + * @property {(prompt: string, longTerm: string | null | undefined, opts: { freshSession: boolean }) => string} [memoryPreamble] default: the prompt unchanged + * @property {(trust: string | undefined) => string | undefined} [effectiveTrust] default: the declared trust unchanged + * @property {() => boolean} [seatModelRequired] default: false + */ + +/** + * @param {Adapter[]} adapters + */ +export const createRegistry = (adapters) => { + const byName = {}; + for (const adapter of adapters) { + if (!adapter?.name) throw new Error('every adapter needs a name'); + if (byName[adapter.name]) throw new Error(`duplicate adapter name: ${adapter.name}`); + byName[adapter.name] = adapter; + } + return { + adapters: byName, + names: () => Object.keys(byName), + get: (name) => byName[name] || null, + }; +}; + +export const providerKeyEnvNames = (adapters) => [...new Set( + Object.values(adapters).map((adapter) => adapter.providerKeyEnv).filter(Boolean), +)]; diff --git a/packages/seats/src/failures.js b/packages/seats/src/failures.js new file mode 100644 index 0000000000..046a39c852 --- /dev/null +++ b/packages/seats/src/failures.js @@ -0,0 +1,38 @@ +/** + * Named run failures. Every way a run can end badly has one kind, so a trace, + * a retry policy and a dashboard all read the same word. A kind is a contract: + * add one here, never mint one at a call site. + */ +import { scrubAdapterFailure } from './upstream-refusal.js'; + +export const FAILURE_KINDS = Object.freeze({ + stuck: { retryable: true, meaning: 'no progress signal within the liveness window' }, + timeout: { retryable: true, meaning: 'the run exceeded its wall-clock budget' }, + exited: { retryable: true, meaning: 'the CLI process ended before the turn settled' }, + protocol: { retryable: false, meaning: 'the CLI broke its wire protocol' }, + refusal: { retryable: false, meaning: 'the model or provider refused; never retried' }, + max_tokens: { retryable: false, meaning: 'the turn hit its token limit' }, + max_turn_requests: { retryable: false, meaning: 'the turn hit its request limit' }, + cancelled: { retryable: false, meaning: 'the run was cancelled' }, + credential_denied: { retryable: false, meaning: 'the run asked for a credential its grant does not allow' }, + unsupported: { retryable: false, meaning: 'the CLI or its version is outside what this adapter supports' }, +}); + +export class RunFailure extends Error { + constructor(kind, detail, { cause } = {}) { + if (!FAILURE_KINDS[kind]) throw new Error(`unknown failure kind: ${kind}`); + super(detail ? `${kind}: ${detail}` : kind, cause ? { cause } : undefined); + this.name = 'RunFailure'; + this.kind = kind; + this.retryable = FAILURE_KINDS[kind].retryable; + this.detail = detail || ''; + } +} + +/** Build a failure whose detail has known credentials and secret shapes scrubbed. */ +export const runFailure = (kind, text, { credentials = [], limit } = {}) => { + const { detail } = scrubAdapterFailure(String(text ?? ''), { credentials, limit }); + return new RunFailure(kind, detail); +}; + +export const failureKindOf = (error) => (error instanceof RunFailure ? error.kind : null); diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js new file mode 100644 index 0000000000..47e30b6def --- /dev/null +++ b/packages/seats/src/index.js @@ -0,0 +1,22 @@ +export { createRegistry, providerKeyEnvNames } from './contract.js'; +export { requireSeatModel } from './seat-model.js'; +export { controlsFromArguments, observeSpawn } from './runtime-observation.js'; +export { + LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP, spawnInOwnGroup, wantsOwnProcessGroup, leadsOwnGroup, killProcessTree, + trackSupervisedGroup, killSupervisedGroups, armSupervisedGroupDeadline, + readProcessTable, readProcess, descendantsOf, killDescendants, +} from './process-group.js'; +export { adapterFailure, spawnCredentials } from './upstream-refusal.js'; +export { default as stub } from './adapters/stub.js'; +export { createHermesAdapter } from './adapters/hermes.js'; +export { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; +export { EVENT_TYPES, isEventType, traceOf, NULL_TRACE } from './trace/events.js'; +export { createTraceStore } from './trace/store.js'; +export { validateGrant, matchToolCall, TOOL_KINDS, CREDENTIAL_SOURCES } from './valet/grant.js'; +export { prepareRunEnv, BASE_ENV_ALLOWLIST } from './valet/env.js'; +export { createPermissionPolicy, DEFAULT_APPROVE_TIMEOUT_MS } from './valet/permissions.js'; +export { + RECORD_VERSION, RECORD_SCHEMA, KINDS, canonicalLine, appendEntry, verifyRecord, readRecord, hashLine, toolSurfaceOf, argsDigestOf, +} from './record/format.js'; +export { toOtlpJson } from './record/otel.js'; +export { superviseRun, SUPERVISOR_DEFAULTS, DEFAULT_RETRY_ON } from './supervisor.js'; diff --git a/packages/seats/src/process-group.js b/packages/seats/src/process-group.js new file mode 100644 index 0000000000..286b856e23 --- /dev/null +++ b/packages/seats/src/process-group.js @@ -0,0 +1,261 @@ +/** + * Process groups for supervised children. + * + * An agent CLI spawns its own children: MCP servers through `npx`, a native + * binary behind a node launcher, shells for tools. Killing only the CLI leaves + * those running, holding the run's work and keys on a shared host. A child the + * supervisor will kill is therefore started as the leader of its own process + * group, and killed as a group. + * + * Leading a group is never inferred. The spawner that passed `detached` marks + * the child, and only a marked child that is still alive is ever signalled by + * negative pid: signalling `-pid` for a process that does not lead its group + * can reach an unrelated group. + * + * A group kill misses a descendant that moved to a group of its own (codex + * starts every MCP server and tool command that way; claude's Bash tool does + * too). So before any signal, the kill also snapshots the process table and + * walks the child's descendants, deepest first. Each pid is re-read right + * before it is signalled and killed only if its parent and start time still + * match the snapshot: a pid that exited and was reused by another process on + * this host is left alone. Leaves go first so a pid's parent is still alive, + * and its ppid still meaningful, when it is checked. + * + * What the walk does not catch, by design, because each case would need the + * ppid match loosened: + * (a) a descendant reparented to 1 (its parent died) between the snapshot + * and its re-read is skipped: its ppid no longer matches; + * (b) a process started after the snapshot is not in it, and one that calls + * setsid after the snapshot leaves the group the group kill signals; + * either is missed. + * + * A child in its own group is outside the wrapper's group, so launchd's kill + * of the seat's group on exit no longer reaches it. `trackSupervisedGroup` + * covers that: while any tracked group is live, the wrapper kills them all + * when it exits, and on SIGTERM, SIGINT or SIGHUP when nothing else handles + * the signal. A host that handles the signal itself (`agent run` finishes the + * in-flight turn on SIGTERM) owns the timing: it calls + * `armSupervisedGroupDeadline` so the groups die before launchd's SIGKILL, + * which no handler can catch, and the exit hook covers the clean case. + */ + +import { spawnSync } from 'node:child_process'; + +/** Set on a child by the code that spawned it with `detached: true`. */ +export const LEADS_PROCESS_GROUP = Symbol.for('@commonlyai/seats.leadsProcessGroup'); + +/** Whether a child can lead its own group here (POSIX only). */ +export const PROCESS_GROUPS_SUPPORTED = process.platform !== 'win32'; + +/** + * The ctx key the supervisor sets to tell an adapter it will kill the child's + * group, so the adapter may start the child in one. Unset, an adapter spawns + * exactly as it would unsupervised. + */ +export const OWN_PROCESS_GROUP = 'ownProcessGroup'; + +export const wantsOwnProcessGroup = (ctx) => ctx?.[OWN_PROCESS_GROUP] === true && PROCESS_GROUPS_SUPPORTED; + +/** + * Spawn through `spawnImpl`. With `ownGroup`, the child is started detached + * (it calls setsid, so it leads a new group whose id is its pid) and marked. + * Without it, `options` reach `spawnImpl` untouched. + */ +export const spawnInOwnGroup = (spawnImpl, command, args, options, { ownGroup = false } = {}) => { + if (!ownGroup || !PROCESS_GROUPS_SUPPORTED) return spawnImpl(command, args, options); + const child = spawnImpl(command, args, { ...options, detached: true }); + if (child && typeof child === 'object') child[LEADS_PROCESS_GROUP] = true; + return child; +}; + +const isAlive = (child) => child.exitCode === null && child.signalCode === null; + +/** + * True only for a child its spawner marked, with a real pid, that has not been + * reaped. Node sets exitCode/signalCode in the same tick it reaps, so while + * this holds the pid (and so the group id) cannot have been reused. + */ +export const leadsOwnGroup = (child) => !!child + && child[LEADS_PROCESS_GROUP] === true + && Number.isInteger(child.pid) && child.pid > 0 + && isAlive(child); + +// ── The process table ────────────────────────────────────────────────────── + +const PS_ENV = { ...process.env, LC_ALL: 'C' }; + +const parseTable = (text) => { + const rows = []; + for (const line of String(text || '').split('\n')) { + const match = /^\s*(\d+)\s+(\d+)\s+(\S.*?)\s*$/.exec(line); + if (match) rows.push({ pid: Number(match[1]), ppid: Number(match[2]), started: match[3] }); + } + return rows; +}; + +const ps = (args) => { + try { + const out = spawnSync('ps', args, { encoding: 'utf8', env: PS_ENV, timeout: 5000 }); + return out.error ? null : parseTable(out.stdout); + } catch { return null; } +}; + +/** Every process: { pid, ppid, started }. null when ps is unavailable. */ +export const readProcessTable = () => ps(['-A', '-o', 'pid=,ppid=,lstart=']); + +/** One process as it is now, or null when it is gone. */ +export const readProcess = (pid) => (ps(['-o', 'pid=,ppid=,lstart=', '-p', String(pid)]) || []) + .find((row) => row.pid === pid) || null; + +/** The descendants of `root` in `table`, deepest first. The root is not included. */ +export const descendantsOf = (table, root) => { + const byParent = new Map(); + for (const row of table || []) { + if (row.pid === row.ppid) continue; + if (!byParent.has(row.ppid)) byParent.set(row.ppid, []); + byParent.get(row.ppid).push(row); + } + const out = []; + const seen = new Set([root]); + let level = [root]; + for (let depth = 1; level.length; depth += 1) { + const next = []; + for (const parent of level) { + for (const row of byParent.get(parent) || []) { + if (seen.has(row.pid)) continue; + seen.add(row.pid); + out.push({ ...row, depth }); + next.push(row.pid); + } + } + level = next; + } + return out.sort((a, b) => b.depth - a.depth); +}; + +const DEFAULT_IO = Object.freeze({ table: readProcessTable, read: readProcess }); + +/** + * Signal each snapshotted descendant, deepest first, re-reading it just + * before: it is signalled only while its ppid and start time still match. + * Returns the pids signalled. + */ +export const killDescendants = (descendants, signal = 'SIGKILL', { read = readProcess } = {}) => { + const killed = []; + for (const row of descendants) { + const now = read(row.pid); + if (!now || now.ppid !== row.ppid || now.started !== row.started) continue; + try { process.kill(row.pid, signal); killed.push(row.pid); } catch { /* gone */ } + } + return killed; +}; + +const liveRoot = (child) => !!child && Number.isInteger(child.pid) && child.pid > 0 && isAlive(child); + +const killOne = (child, signal, table, io) => { + // The root must be alive, so unreaped, so its pid still names this child. + if (table && liveRoot(child)) killDescendants(descendantsOf(table, child.pid), signal, io); + if (leadsOwnGroup(child)) { + try { process.kill(-child.pid, signal); return true; } catch { /* fall back to the leader */ } + } + try { return child.kill(signal) !== false; } catch { return false; } +}; + +/** + * Kill a child, its verified descendants, and its whole group when it leads + * one. The table is read before any signal. Falls back to `child.kill` when + * the child does not lead a group, or the group signal fails. Returns whether + * the child was signalled. `io` is a test seam ({ table, read }). + */ +export const killProcessTree = (child, signal = 'SIGKILL', io = {}) => { + const seam = { ...DEFAULT_IO, ...io }; + return killOne(child, signal, liveRoot(child) ? seam.table() : null, seam); +}; + +// ── Wrapper-exit cleanup ──────────────────────────────────────────────────── + +const EXIT_SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP']; +const tracked = new Set(); +const handlers = new Map(); +let onExit = null; +let deadline = null; + +/** SIGKILL every tracked group still led by a live child, with its verified descendants. Synchronous. */ +export const killSupervisedGroups = (signal = 'SIGKILL', io = {}) => { + const live = [...tracked].filter(leadsOwnGroup); + if (live.length === 0) return 0; + const seam = { ...DEFAULT_IO, ...io }; + const table = seam.table(); + let killed = 0; + for (const child of live) if (killOne(child, signal, table, seam)) killed += 1; + return killed; +}; + +/** + * Kill the tracked groups in `ms`, for a host that keeps running after a + * signal (to finish a turn) but will be SIGKILLed at a known deadline. The + * earliest deadline armed wins. The timer does not hold the process open. + * Returns a function that disarms it. + */ +export const armSupervisedGroupDeadline = (ms) => { + const wait = Math.max(0, Number(ms) || 0); + const at = Date.now() + wait; + if (deadline && deadline.at <= at) return deadline.cancel; + if (deadline) deadline.cancel(); + const timer = setTimeout(() => { deadline = null; killSupervisedGroups(); }, wait); + timer.unref?.(); + const entry = { at, cancel: () => { clearTimeout(timer); if (deadline === entry) deadline = null; } }; + deadline = entry; + return entry.cancel; +}; + +const uninstall = () => { + for (const [signal, handler] of handlers) process.removeListener(signal, handler); + handlers.clear(); + if (onExit) process.removeListener('exit', onExit); + onExit = null; +}; + +const install = () => { + if (onExit) return; + onExit = () => { killSupervisedGroups(); }; + process.on('exit', onExit); + for (const signal of EXIT_SIGNALS) { + const handler = () => { + // A host that handles this signal has taken over shutdown: it may be + // finishing the turn these groups belong to, so they are left alone and + // the host's deadline or exit kills them. + if (process.listenerCount(signal) !== 1) return; + killSupervisedGroups(); + // Listening for a signal replaces its default action. With nothing else + // handling it, restore that action and re-raise, so the wrapper still + // terminates exactly as it would have without this module. + uninstall(); + process.kill(process.pid, signal); + }; + handlers.set(signal, handler); + process.on(signal, handler); + } +}; + +const untrack = (child) => { + tracked.delete(child); + if (tracked.size === 0) uninstall(); +}; + +/** + * Track a group-leading child until it exits. The handlers exist only while a + * tracked child is live, so a wrapper with no supervised group in flight keeps + * its signal behaviour unchanged. A child that does not lead a group is + * ignored: launchd's group kill already reaches it. + */ +export const trackSupervisedGroup = (child) => { + if (!leadsOwnGroup(child) || tracked.has(child)) return false; + tracked.add(child); + install(); + child.once?.('exit', () => untrack(child)); + return true; +}; + +/** For tests: how many groups are tracked. */ +export const trackedGroupCount = () => tracked.size; diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js new file mode 100644 index 0000000000..2612d8a91c --- /dev/null +++ b/packages/seats/src/record/format.js @@ -0,0 +1,635 @@ +/** + * The run record, format v1: one canonical JSON object per line, each line + * hash-chained to the one before it. docs/record-v1.md is the specification; + * this file is its reference writer and verifier, and nothing here imports + * the store, so another tool can vendor it alone. + */ +import { createHash } from 'node:crypto'; +import { isIP } from 'node:net'; +import { FAILURE_KINDS } from '../failures.js'; + +export const RECORD_VERSION = 1; +export const RECORD_SCHEMA = 'seats.record'; + +export const ACTOR_TYPES = Object.freeze(['agent', 'human', 'policy', 'system']); +// Who decides an approval today is the owner of the credential it would use; +// `decision-owner` is reserved for a separately assigned decider. +export const DECIDED_BY_ROLES = Object.freeze(['credential-owner', 'decision-owner', 'agent-owner', 'pod-member', 'policy']); +// A parked tool call is decided by an owner or by policy, never by a pod member +// who owns neither the credential nor the asking agent (ADR-033 D5). +export const PARKED_CALL_ROLES = Object.freeze(DECIDED_BY_ROLES.filter((role) => role !== 'pod-member')); +export const TOOL_SURFACES = Object.freeze(['connector', 'commonly', 'native']); +export const GRANT_ENDS_ON = Object.freeze(['task.done', 'expiresAt']); +export const REVOKE_REASONS = Object.freeze(['task.done', 'expired', 'cascade', 'manual']); +export const WRITE_MODES = Object.freeze(['read', 'write', 'write-with-confirm']); +export const GRANT_TARGET_KINDS = Object.freeze(['pod', 'seat']); +export const OUTCOMES = Object.freeze(['ok', 'failed', 'stuck', 'cancelled']); +export const RESULT_STATUSES = Object.freeze(['done', 'partial', 'blocked', 'failed']); +// Whose request an agent acts on: 'requester' sent the message that started +// the run, 'assigner' assigned its task. Absent when no human asked. +export const ON_BEHALF_ROLES = Object.freeze(['requester', 'assigner']); +// Why the egress proxy refused a request. The first two can leave host and port null. +export const EGRESS_DENIED_REASONS = Object.freeze([ + 'bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable', +]); +const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); +export const SANDBOXES = Object.freeze(['seatbelt', 'bwrap', 'none']); +export const SESSION_METHODS = Object.freeze(['new', 'resume', 'load']); +export const RETRY_STATES = Object.freeze(['scheduled', 'started']); +/** + * agent.retry.errorType. The provider error types opencode 2.1.4 reports on a + * retry, read from its error formatter, plus `other` for any type outside + * them. A closed set, so no agent text can ride in this field. + */ +export const RETRY_ERROR_TYPES = Object.freeze([ + 'provider.auth', + 'provider.content-filter', + 'provider.free-limit', + 'provider.internal', + 'provider.invalid-output', + 'provider.invalid-request', + 'provider.no-route', + 'provider.quota', + 'provider.rate-limit', + 'provider.timeout', + 'provider.transport', + 'provider.unknown', + 'provider.unsupported-operation', + 'other', +]); +// Why a requested session was not reopened: the two forms an ACP adapter +// writes. The code is the JSON-RPC error's, an integer by JSON-RPC 2.0. +const SESSION_FALLBACK = /^(?:agent supports neither session\/resume nor session\/load|session\/(?:resume|load) answered error -?\d{1,16})$/; +export const REDACTED = '[redacted]'; + +/** The actors a writer records for its own bookkeeping, and for the board. */ +export const SYSTEM_ACTOR = Object.freeze({ type: 'system', id: 'seats' }); +export const BOARD_ACTOR = Object.freeze({ type: 'system', id: 'board' }); + +/** + * Every v1 kind. `actor` is who a writer records when the emitter names + * nobody: 'run' (the run's own actor), 'system', or derived from the data + * ('issuer', 'decider'). `trace` is the EVENT_TYPES name, when it differs. + */ +export const KINDS = Object.freeze({ + 'task.claimed': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.released': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.done': Object.freeze({ group: 'claim', actor: 'board' }), + 'grant.issued': Object.freeze({ group: 'grant', actor: 'issuer' }), + 'grant.revoked': Object.freeze({ group: 'grant', actor: 'system' }), + 'permission.requested': Object.freeze({ group: 'approval', actor: 'run' }), + 'permission.decided': Object.freeze({ group: 'approval', actor: 'decider' }), + 'decision.ruled': Object.freeze({ group: 'approval', actor: 'decider' }), + 'attempt.started': Object.freeze({ group: 'attempt', actor: 'run', trace: 'run.started' }), + 'session.opened': Object.freeze({ group: 'attempt', actor: 'system' }), + 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + 'tool.call': Object.freeze({ group: 'tool', actor: 'run' }), + 'tool.update': Object.freeze({ group: 'tool', actor: 'run' }), + 'credential.granted': Object.freeze({ group: 'credential', actor: 'system' }), + 'credential.denied': Object.freeze({ group: 'credential', actor: 'system' }), + 'result.reported': Object.freeze({ group: 'result', actor: 'run' }), + 'turn.prompt_sent': Object.freeze({ group: 'activity', actor: 'system' }), + 'agent.message_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.thought_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.retry': Object.freeze({ group: 'activity', actor: 'run' }), + plan: Object.freeze({ group: 'activity', actor: 'run' }), + usage: Object.freeze({ group: 'activity', actor: 'run' }), + 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), + 'liveness.recovery': Object.freeze({ group: 'activity', actor: 'system' }), + 'process.exited': Object.freeze({ group: 'activity', actor: 'system' }), + // The run's confinement, recorded once per spawn, beside its egress. + 'sandbox.applied': Object.freeze({ group: 'sandbox', actor: 'system' }), + 'egress.allowed': Object.freeze({ group: 'egress', actor: 'system' }), + 'egress.denied': Object.freeze({ group: 'egress', actor: 'system' }), +}); + +const TRACE_TO_KIND = new Map(Object.entries(KINDS).filter(([, spec]) => spec.trace).map(([kind, spec]) => [spec.trace, kind])); + +/** Record kind for an EVENT_TYPES name or a kind; null when it is neither. */ +export const kindOfType = (type) => TRACE_TO_KIND.get(type) || (Object.hasOwn(KINDS, type) ? type : null); +/** The EVENT_TYPES name readRun reports for a kind (unknown kinds pass through). */ +export const traceTypeOf = (kind) => KINDS[kind]?.trace || kind; + +const fail = (message) => { throw new Error(`record: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isText = (value) => typeof value === 'string' && value.length > 0; +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/; +const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; +// A user id is an opaque token from the instance: no spaces, so a display +// name can't pass for one. Display names go in actor.name. +const USER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +export const isUserId = (value) => typeof value === 'string' && USER_ID.test(value); +const TOOL_NAME = /^[^\s*?]{1,256}$/; +const RFC3339_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +const KIND_SHAPE = /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$/; +const isTimestamp = (value) => typeof value === 'string' && RFC3339_UTC.test(value) && !Number.isNaN(Date.parse(value)); + +/** + * RFC 8785 (JCS) serialization. For the JSON values a record holds, JCS is + * JSON.stringify with object keys sorted by UTF-16 code units, which is what + * Array.prototype.sort does by default. + * + * `strict` also refuses a lone surrogate in any string or key, as RFC 8785 + * requires. Lines are not strict: agent text from a stream cut mid-pair must + * still record, and JSON.stringify writes the lone half as a \u escape. + */ +const canonicalString = (text, strict) => { + if (strict && !text.isWellFormed()) fail('strings must be well-formed UTF-16 (no lone surrogates)'); + return JSON.stringify(text); +}; +const canonical = (value, strict = false) => { + if (value !== null && typeof value === 'object' && typeof value.toJSON === 'function') return canonical(value.toJSON(), strict); + if (value === null) return 'null'; + switch (typeof value) { + case 'string': return canonicalString(value, strict); + case 'boolean': return value ? 'true' : 'false'; + case 'number': + if (!Number.isFinite(value)) fail('numbers must be finite'); + return JSON.stringify(value); + case 'object': + if (Array.isArray(value)) return `[${value.map((item) => (item === undefined ? 'null' : canonical(item, strict))).join(',')}]`; + return `{${Object.keys(value).filter((key) => value[key] !== undefined).sort() + .map((key) => `${canonicalString(key, strict)}:${canonical(value[key], strict)}`).join(',')}}`; + default: return fail(`cannot serialize a ${typeof value}`); + } +}; + +/** The canonical line for an entry, without its trailing newline. */ +export const canonicalLine = (entry) => { + if (!isPlainObject(entry)) fail('an entry is a JSON object'); + return canonical(entry); +}; + +/** Lowercase hex SHA-256 over a line's UTF-8 bytes: what the next `prev` holds. */ +export const hashLine = (line) => createHash('sha256').update(String(line), 'utf8').digest('hex'); + +/** + * A tool call's arguments as a digest, never the values: `sha256:` + the + * lowercase hex SHA-256 of the arguments' RFC 8785 form. The same arguments + * give the same digest wherever they are seen, so a reader can match a + * tool.call to the permission entries that approved it. Returns undefined + * when there are no arguments, or they are not JSON (a function, NaN, a + * string or key holding a lone surrogate): a writer then omits argsDigest. + */ +const ARGS_DIGEST = /^sha256:[0-9a-f]{64}$/; +export const argsDigestOf = (args) => { + if (args === undefined) return undefined; + try { return `sha256:${hashLine(canonical(args, true))}`; } catch { return undefined; } +}; +export const isArgsDigest = (value) => typeof value === 'string' && ARGS_DIGEST.test(value); + +/** null when the actor is well formed, else the reason. */ +const actorProblem = (actor) => { + if (actor === null) return null; + if (!isPlainObject(actor)) return 'actor must be an object or null'; + if (!ACTOR_TYPES.includes(actor.type)) return `actor.type must be one of ${ACTOR_TYPES.join(', ')}`; + if (!isUserId(actor.id)) return 'actor.id must be an id token (a user id for agent and human actors), never a display name'; + if (actor.owner !== undefined && (actor.type !== 'agent' || !isUserId(actor.owner))) return 'actor.owner is a user id, on agent actors only'; + if (actor.name !== undefined && !isText(actor.name)) return 'actor.name must be a non-empty string'; + if (actor.onBehalfOf !== undefined) { + const behalf = actor.onBehalfOf; + if (actor.type !== 'agent') return 'actor.onBehalfOf is on agent actors only'; + if (!isPlainObject(behalf)) return 'actor.onBehalfOf must be {id, role}'; + const extra = Object.keys(behalf).find((key) => key !== 'id' && key !== 'role'); + if (extra !== undefined) return `unknown field actor.onBehalfOf.${extra}`; + if (!isUserId(behalf.id)) return 'actor.onBehalfOf.id must be an id token (a user id), never a display name'; + if (!ON_BEHALF_ROLES.includes(behalf.role)) return `actor.onBehalfOf.role must be one of ${ON_BEHALF_ROLES.join(', ')}`; + } + return null; +}; + +export const validateActor = (actor) => { + const problem = actorProblem(actor); + if (problem) fail(problem); + if (actor === null) return null; + return Object.freeze({ + type: actor.type, + id: actor.id, + ...(actor.owner ? { owner: actor.owner } : {}), + ...(actor.onBehalfOf ? { onBehalfOf: Object.freeze({ id: actor.onBehalfOf.id, role: actor.onBehalfOf.role }) } : {}), + ...(actor.name ? { name: actor.name } : {}), + }); +}; + +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); +}; +const envNames = (names, where) => { + if (!Array.isArray(names) || !names.every((name) => typeof name === 'string' && ENV_NAME.test(name))) { + fail(`${where} must be credential NAMES ([A-Z_][A-Z0-9_]*), never values`); + } + return [...names]; +}; +const optionalText = (value, where) => { + if (value !== undefined && value !== null && !isText(value)) fail(`${where} must be a non-empty string`); +}; +const userIdOrPolicy = (value, where) => { + if (value !== undefined && value !== null && value !== 'policy' && !isUserId(value)) fail(`${where} must be a user id or "policy"`); +}; +// Only the digest form: raw arguments would carry values. +const optionalArgsDigest = (data, kind) => { + if (data.argsDigest !== undefined && !isArgsDigest(data.argsDigest)) fail(`${kind}.argsDigest must be sha256:<64 hex>, a digest of the arguments, never the arguments`); +}; +// A host as the egress proxy traces it: a lowercase DNS name, an IP literal +// (IPv6 without brackets), or [redacted] when it held a run credential. +const DNS_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; +const isEgressHost = (host) => typeof host === 'string' && (host === REDACTED || isIP(host) !== 0 + || (host.length <= 253 && host.split('.').every((label) => DNS_LABEL.test(label)))); +const isPort = (port) => Number.isInteger(port) && port >= 1 && port <= 65535; +const checkEgressTarget = (data, kind, { nullable }) => { + if (data.host === null && data.port === null && nullable) return; + if (data.host === null || data.port === null) { + fail(nullable ? `${kind}: host and port are null together, or both set` : `${kind} needs a host and a port`); + } + if (!isEgressHost(data.host)) fail(`${kind}.host must be a lowercase DNS name, an IP literal or "${REDACTED}"`); + if (!isPort(data.port)) fail(`${kind}.port must be an integer from 1 to 65535`); +}; + +const checkRole = (role, where, roles = DECIDED_BY_ROLES) => { + if (!roles.includes(role)) fail(`${where} must be one of ${roles.join(', ')}`); +}; + +/** A grant's scope, in the shape of a room grant: exact tool names and a connection id. */ +const checkScope = (scope) => { + if (!isPlainObject(scope)) fail('grant.issued needs a scope'); + refuseUnknownKeys(scope, ['connectionId', 'tools', 'writeMode', 'budget', 'audience'], 'grant.issued.scope.'); + optionalText(scope.connectionId, 'grant.issued.scope.connectionId'); + const tools = scope.tools ?? []; + if (!Array.isArray(tools) || !tools.every((name) => typeof name === 'string' && TOOL_NAME.test(name))) { + fail('grant.issued.scope.tools must be exact tool names (no spaces, no * or ?), not patterns'); + } + if (scope.writeMode !== undefined && !WRITE_MODES.includes(scope.writeMode)) fail(`grant.issued.scope.writeMode must be one of ${WRITE_MODES.join(', ')}`); + let budget; + if (scope.budget !== undefined) { + if (!isPlainObject(scope.budget)) fail('grant.issued.scope.budget must be an object'); + refuseUnknownKeys(scope.budget, ['calls', 'windowMs'], 'grant.issued.scope.budget.'); + if (scope.budget.calls !== undefined && !(Number.isInteger(scope.budget.calls) && scope.budget.calls >= 0)) fail('grant.issued.scope.budget.calls must be an integer >= 0'); + if (scope.budget.windowMs !== undefined && !(Number.isInteger(scope.budget.windowMs) && scope.budget.windowMs >= 1)) fail('grant.issued.scope.budget.windowMs must be an integer >= 1'); + budget = { ...scope.budget }; + } + const audience = scope.audience ?? []; + if (!Array.isArray(audience) || !audience.every(isUserId)) fail('grant.issued.scope.audience must be user ids'); + return { + ...(scope.connectionId ? { connectionId: scope.connectionId } : {}), + tools: [...tools], + ...(scope.writeMode ? { writeMode: scope.writeMode } : {}), + ...(budget ? { budget } : {}), + audience: [...audience], + }; +}; + +// Per-kind checks, applied by every writer. They are strictest where a value +// could ride in: a grant's scope, target and binding refuse fields they don't define. +const CHECKS = { + 'grant.issued': (data, envelope) => { + refuseUnknownKeys(data, ['grantId', 'parentGrantId', 'scope', 'target', 'issuedBy', 'expiresAt', 'binding'], 'grant.issued.'); + if (!isText(data.grantId)) fail('grant.issued needs a grantId'); + optionalText(data.parentGrantId, 'grant.issued.parentGrantId'); + const scope = checkScope(data.scope); + let target; + if (data.target !== undefined) { + if (!isPlainObject(data.target)) fail('grant.issued.target must be an object'); + refuseUnknownKeys(data.target, ['kind', 'id'], 'grant.issued.target.'); + if (!GRANT_TARGET_KINDS.includes(data.target.kind) || !isText(data.target.id)) fail('grant.issued.target is {kind: pod|seat, id}'); + target = { kind: data.target.kind, id: data.target.id }; + } + userIdOrPolicy(data.issuedBy, 'grant.issued.issuedBy'); + if (data.expiresAt !== undefined && !isTimestamp(data.expiresAt)) fail('grant.issued.expiresAt must be an RFC 3339 UTC timestamp'); + let binding; + if (data.binding !== undefined) { + if (!isPlainObject(data.binding)) fail('grant.issued.binding must be an object'); + refuseUnknownKeys(data.binding, ['taskId', 'podId', 'endsOn'], 'grant.issued.binding.'); + if (!isText(data.binding.taskId) || !isText(data.binding.podId)) fail('grant.issued.binding needs taskId and podId: task ids repeat across pods'); + if (data.binding.taskId !== envelope.taskId || data.binding.podId !== envelope.podId) fail('grant.issued.binding must name the run\'s own taskId and podId'); + if (!GRANT_ENDS_ON.includes(data.binding.endsOn)) fail(`grant.issued.binding.endsOn must be one of ${GRANT_ENDS_ON.join(', ')}`); + if (data.binding.endsOn === 'expiresAt' && data.expiresAt === undefined) fail('a grant that ends on expiresAt needs an expiresAt'); + binding = { taskId: data.binding.taskId, podId: data.binding.podId, endsOn: data.binding.endsOn }; + } + return { + ...data, + ...(data.parentGrantId === undefined ? {} : { parentGrantId: data.parentGrantId }), + scope, + ...(target ? { target } : {}), + ...(binding ? { binding } : {}), + }; + }, + 'grant.revoked': (data) => { + if (!Array.isArray(data.grantIds) || data.grantIds.length === 0 || !data.grantIds.every(isText)) fail('grant.revoked needs grantIds: a non-empty array'); + if (new Set(data.grantIds).size !== data.grantIds.length) fail('grant.revoked.grantIds repeats an id'); + if (!REVOKE_REASONS.includes(data.reason)) fail(`grant.revoked.reason must be one of ${REVOKE_REASONS.join(', ')}`); + return { ...data, grantIds: [...data.grantIds] }; + }, + 'task.done': (data, envelope) => { + if (!envelope.taskId) fail('task.done needs a run with a taskId'); + userIdOrPolicy(data.doneBy, 'task.done.doneBy'); + return data; + }, + 'permission.requested': (data) => { + if (!isText(data.question)) fail('permission.requested needs a question: the human-readable ask'); + if (!isPlainObject(data.askedOf)) fail('permission.requested needs askedOf: {id, role}'); + refuseUnknownKeys(data.askedOf, ['id', 'role'], 'permission.requested.askedOf.'); + if (!isUserId(data.askedOf.id)) fail('permission.requested.askedOf.id must be an id token, never a display name'); + checkRole(data.askedOf.role, 'permission.requested.askedOf.role', PARKED_CALL_ROLES); + optionalText(data.grantId, 'permission.requested.grantId'); + optionalText(data.decisionCardId, 'permission.requested.decisionCardId'); + optionalArgsDigest(data, 'permission.requested'); + return { ...data, askedOf: { id: data.askedOf.id, role: data.askedOf.role } }; + }, + 'decision.ruled': (data) => { + if (!isText(data.decisionCardId)) fail('decision.ruled needs a decisionCardId'); + const { value } = data; + if (!(typeof value === 'string' || typeof value === 'boolean' || (typeof value === 'number' && Number.isFinite(value)))) { + fail('decision.ruled.value must be a string, number or boolean'); + } + if (data.decidedBy !== 'policy' && !isUserId(data.decidedBy)) fail('decision.ruled.decidedBy must be a user id or "policy"'); + checkRole(data.decidedByRole, 'decision.ruled.decidedByRole'); + return data; + }, + 'permission.decided': (data) => { + optionalArgsDigest(data, 'permission.decided'); + // A rule-decided call never had a person in the loop; a writer says so + // rather than leave a reader to infer it from rule.list. + // A decision always names who made it: no writer may record one nobody + // can be held to. + const out = { ...data }; + if (!('decidedBy' in out) && !('decidedByRole' in out) && out.rule && out.rule.list !== 'ask') { + out.decidedBy = 'policy'; + out.decidedByRole = 'policy'; + } + if (out.decidedBy === undefined || out.decidedBy === null) fail('permission.decided.decidedBy is required: a user id, or "policy"'); + userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); + checkRole(out.decidedByRole, 'permission.decided.decidedByRole', PARKED_CALL_ROLES); + if ((out.decidedBy === 'policy') !== (out.decidedByRole === 'policy')) { + fail('permission.decided.decidedBy is "policy" exactly when decidedByRole is "policy"'); + } + optionalText(out.decisionCardId, 'permission.decided.decisionCardId'); + return out; + }, + 'tool.call': (data) => { + if (data.surface !== undefined && !TOOL_SURFACES.includes(data.surface)) fail(`tool.call.surface must be one of ${TOOL_SURFACES.join(', ')}`); + optionalArgsDigest(data, 'tool.call'); + return data; + }, + 'credential.granted': (data) => ({ ...data, names: envNames(data.names ?? [], 'credential.granted.names') }), + 'credential.denied': (data) => { + envNames([data.name], 'credential.denied.name'); + return data; + }, + 'attempt.settled': (data) => { + if (!Number.isInteger(data.attempt) || data.attempt < 1) fail('attempt.settled.attempt must be a positive integer'); + if (!OUTCOMES.includes(data.outcome)) fail(`attempt.settled.outcome must be one of ${OUTCOMES.join(', ')}`); + if (data.failure !== undefined && data.failure !== null) { + const { kind } = data.failure; + if (kind !== null && kind !== undefined && !FAILURE_KINDS[kind]) fail(`attempt.settled.failure.kind ${kind} is not a FAILURE_KINDS key`); + } + return data; + }, + 'session.opened': (data) => { + refuseUnknownKeys(data, ['method', 'sessionId', 'requestedSessionId', 'fallback'], 'session.opened.'); + if (!SESSION_METHODS.includes(data.method)) fail(`session.opened.method must be one of ${SESSION_METHODS.join(', ')}`); + if (!isText(data.sessionId)) fail('session.opened needs a sessionId'); + if ((data.requestedSessionId === undefined) !== (data.fallback === undefined)) { + fail('session.opened.requestedSessionId and fallback come together: a session was asked for and not reopened'); + } + if (data.requestedSessionId !== undefined) { + if (data.method !== 'new') fail('session.opened.requestedSessionId is set only when method is new'); + if (!isText(data.requestedSessionId)) fail('session.opened.requestedSessionId must be a non-empty string'); + if (typeof data.fallback !== 'string' || !SESSION_FALLBACK.test(data.fallback)) { + fail('session.opened.fallback must be "agent supports neither session/resume nor session/load" or "session/ answered error "'); + } + } + return data; + }, + 'agent.retry': (data) => { + refuseUnknownKeys(data, ['state', 'attempt', 'nextRetryAt', 'status', 'errorType'], 'agent.retry.'); + if (!RETRY_STATES.includes(data.state)) fail(`agent.retry.state must be one of ${RETRY_STATES.join(', ')}`); + // Every other field is absent when unknown, never null. + if ('attempt' in data && !(Number.isInteger(data.attempt) && data.attempt >= 1)) fail('agent.retry.attempt must be an integer >= 1, or absent'); + if ('nextRetryAt' in data && !isTimestamp(data.nextRetryAt)) fail('agent.retry.nextRetryAt must be an RFC 3339 UTC timestamp, or absent'); + if ('status' in data && !(Number.isInteger(data.status) && data.status >= 100 && data.status <= 599)) fail('agent.retry.status must be an integer HTTP status (100-599), or absent'); + if ('errorType' in data && !RETRY_ERROR_TYPES.includes(data.errorType)) fail(`agent.retry.errorType must be one of ${RETRY_ERROR_TYPES.join(', ')}, or absent`); + return data; + }, + usage: (data) => { + for (const key of ['contextTokens', 'contextSize']) { + const value = data[key]; + if (value !== undefined && value !== null && !(Number.isInteger(value) && value >= 0)) fail(`usage.${key} must be an integer >= 0, or null when unknown`); + } + return data; + }, + 'sandbox.applied': (data) => { + refuseUnknownKeys(data, ['sandbox', 'reason'], 'sandbox.applied.'); + if (!SANDBOXES.includes(data.sandbox)) fail(`sandbox.applied.sandbox must be one of ${SANDBOXES.join(', ')}`); + if (data.reason !== undefined) { + if (data.sandbox !== 'none') fail('sandbox.applied.reason is set only when sandbox is none'); + if (!isText(data.reason)) fail('sandbox.applied.reason must be a non-empty string'); + } + return data; + }, + 'egress.allowed': (data) => { + refuseUnknownKeys(data, ['host', 'port'], 'egress.allowed.'); + checkEgressTarget(data, 'egress.allowed', { nullable: false }); + return data; + }, + 'egress.denied': (data) => { + refuseUnknownKeys(data, ['host', 'port', 'reason'], 'egress.denied.'); + if (!EGRESS_DENIED_REASONS.includes(data.reason)) fail(`egress.denied.reason must be one of ${EGRESS_DENIED_REASONS.join(', ')}`); + checkEgressTarget(data, 'egress.denied', { nullable: EGRESS_TARGETLESS.has(data.reason) }); + return data; + }, + 'result.reported': (data) => { + if (!RESULT_STATUSES.includes(data.status)) fail(`result.reported.status must be one of ${RESULT_STATUSES.join(', ')}`); + if (data.refs !== undefined && !(Array.isArray(data.refs) && data.refs.every((ref) => isPlainObject(ref) && isText(ref.type) && isText(ref.id)))) { + fail('result.reported.refs must be [{type, id}]'); + } + return data; + }, +}; + +/** + * Validate one kind's data and return the copy a writer records. `envelope` + * is {taskId, podId}, for checks that tie data to the run it sits in. + */ +export const normalizeData = (kind, data = {}, envelope = {}) => { + if (!isPlainObject(data)) fail(`${kind} data must be an object`); + const check = CHECKS[kind]; + return check ? check({ ...data }, envelope) : { ...data }; +}; + +/** A grant's rules acting on their own: named by the grant when its id is a token. */ +export const policyActorId = (grantId) => (isText(grantId) && isUserId(`grant:${grantId}`) ? `grant:${grantId}` : 'policy'); +const policyActor = (grantId) => ({ type: 'policy', id: policyActorId(grantId) }); + +/** The actor a writer records when the emitter names none. */ +export const defaultActor = (kind, data = {}, runActor = null) => { + switch (KINDS[kind]?.actor) { + case 'run': return runActor; + case 'system': return SYSTEM_ACTOR; + case 'board': return BOARD_ACTOR; + case 'issuer': + if (data.issuedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.issuedBy) ? { type: 'human', id: data.issuedBy } : null; + case 'decider': + if (data.decidedByRole === 'policy' || data.decidedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.decidedBy) ? { type: 'human', id: data.decidedBy } : null; + default: return null; + } +}; + +const parseLine = (line) => { + let entry; + try { entry = JSON.parse(line); } catch { return fail('the previous line is not JSON'); } + if (!isPlainObject(entry) || !Number.isInteger(entry.seq)) fail('the previous line is not a record entry'); + return entry; +}; + +/** + * appendEntry(prevLine, fields, { runActor }?) → the next line, canonical, + * without its newline. + * + * fields: { runId, taskId?, podId?, ts, kind, actor?, data, seq? }. seq and + * prev come from prevLine (null for the first line); a given seq must agree. + * An absent actor takes the kind's default; null records "unknown". + */ +export const appendEntry = (prevLine, fields = {}, { runActor = null } = {}) => { + let seq = 1; + let prev = null; + if (prevLine !== null && prevLine !== undefined) { + const before = parseLine(prevLine); + seq = before.seq + 1; + prev = hashLine(prevLine); + if (before.runId !== fields.runId) fail('one record holds one run'); + if (before.taskId !== fields.taskId || before.podId !== fields.podId) fail('taskId and podId are fixed for the whole run'); + if (before.kind === 'run.settled') fail('nothing is written after run.settled'); + } + if (fields.seq !== undefined && fields.seq !== seq) fail(`seq ${fields.seq} does not follow ${seq - 1}`); + if (!RUN_ID.test(String(fields.runId))) fail('runId must be 6-64 characters of [A-Za-z0-9_-]'); + optionalText(fields.taskId, 'taskId'); + optionalText(fields.podId, 'podId'); + if (fields.taskId && !fields.podId) fail('a taskId needs a podId: task ids repeat across pods'); + const ts = fields.ts instanceof Date ? fields.ts.toISOString() : fields.ts; + if (!isTimestamp(ts)) fail('ts must be an RFC 3339 UTC timestamp'); + if (!Object.hasOwn(KINDS, fields.kind)) fail(`unknown record kind: ${fields.kind}`); + const data = normalizeData(fields.kind, fields.data ?? {}, { taskId: fields.taskId, podId: fields.podId }); + const actor = validateActor(fields.actor === undefined ? defaultActor(fields.kind, data, runActor) : fields.actor); + return canonicalLine({ + v: RECORD_VERSION, + schema: RECORD_SCHEMA, + runId: fields.runId, + ...(fields.taskId ? { taskId: fields.taskId } : {}), + ...(fields.podId ? { podId: fields.podId } : {}), + seq, + ts, + kind: fields.kind, + actor, + data, + prev, + }); +}; + +/** Split a record file's text into lines; a last line with no newline is torn. */ +export const splitRecord = (text) => { + if (!text) return { lines: [], torn: null }; + const parts = String(text).split('\n'); + const tail = parts.pop(); + return { lines: parts, torn: tail === '' ? null : tail }; +}; + +const envelopeProblem = (entry) => { + if (!Number.isInteger(entry.v) || entry.v < 1) return 'v'; + if (entry.schema !== RECORD_SCHEMA) return 'schema'; + if (!RUN_ID.test(String(entry.runId))) return 'runId'; + if (entry.taskId !== undefined && !isText(entry.taskId)) return 'taskId'; + if (entry.podId !== undefined && !isText(entry.podId)) return 'podId'; + if (entry.taskId !== undefined && entry.podId === undefined) return 'podId'; + if (!Number.isInteger(entry.seq) || entry.seq < 1) return 'seq'; + if (!isTimestamp(entry.ts)) return 'ts'; + if (typeof entry.kind !== 'string' || !KIND_SHAPE.test(entry.kind)) return 'kind'; + if (actorProblem(entry.actor)) return 'actor'; + if (!isPlainObject(entry.data)) return 'data'; + if (entry.prev !== null && !(typeof entry.prev === 'string' && /^[0-9a-f]{64}$/.test(entry.prev))) return 'prev'; + return null; +}; + +/** + * verifyRecord(text | lines) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } + * + * Text is a whole file, so a torn final line (no newline) can be seen; an + * array is taken as complete lines. Each problem is reported at the seq of + * the line where it was observed: + * torn the last line was cut short by a crash mid-write + * unparseable not JSON + * bad_envelope an envelope field is missing or malformed (detail names it) + * unsupported_version v is newer than this reader + * not_canonical the bytes are not the canonical form of what they parse to + * seq_mismatch seq is not the previous seq + 1 (a deleted or moved line) + * prev_mismatch prev is not the hash of the line before it: that line, or + * this one's prev, was changed, removed or moved + * run_mismatch runId, taskId or podId differs from the first line + * after_settled a line follows run.settled + * The chain re-syncs after each problem, so one edit is reported once. + */ +export const verifyRecord = (input) => { + const { lines, torn } = Array.isArray(input) ? { lines: input.map(String), torn: null } : splitRecord(input); + const errors = []; + const report = (seq, reason, detail) => errors.push(detail ? { seq, reason, detail } : { seq, reason }); + let expected = 1; + let prevRaw = null; + let first = null; + let settled = false; + for (const line of lines) { + let entry; + try { entry = JSON.parse(line); } catch { entry = undefined; } + if (!isPlainObject(entry)) { + report(expected, 'unparseable'); + prevRaw = line; + expected += 1; + continue; + } + const seq = Number.isInteger(entry.seq) ? entry.seq : expected; + if (Number.isInteger(entry.v) && entry.v > RECORD_VERSION) report(seq, 'unsupported_version', `v${entry.v}`); + const problem = envelopeProblem(entry); + if (problem) report(seq, 'bad_envelope', problem); + let canonicalForm = null; + try { canonicalForm = canonicalLine(entry); } catch { /* reported below */ } + if (canonicalForm !== line) report(seq, 'not_canonical'); + if (seq !== expected) report(seq, 'seq_mismatch', `expected ${expected}`); + const expectedPrev = prevRaw === null ? null : hashLine(prevRaw); + if (entry.prev !== expectedPrev) report(seq, 'prev_mismatch'); + if (first === null) first = entry; + else if (entry.runId !== first.runId || entry.taskId !== first.taskId || entry.podId !== first.podId) report(seq, 'run_mismatch'); + if (settled) report(seq, 'after_settled'); + if (entry.kind === 'run.settled') settled = true; + prevRaw = line; + expected = seq + 1; + } + if (torn !== null) report(expected, 'torn'); + return { + ok: errors.length === 0, + errors, + entries: lines.length, + head: prevRaw === null ? null : hashLine(prevRaw), + settled, + }; +}; + +/** Parse a record's text into entries, skipping a torn tail and unparseable lines. */ +export const readRecord = (text) => splitRecord(text).lines.flatMap((line) => { + try { + const entry = JSON.parse(line); + return isPlainObject(entry) ? [entry] : []; + } catch { return []; } +}); + +/** + * Which surface a tool name belongs to, for an adapter that knows the real + * tool name (an MCP server/tool pair, say). Never apply it to a title: a + * title is the agent's own text and can claim any prefix. + */ +export const toolSurfaceOf = (toolName) => { + const name = String(toolName || ''); + if (/^(?:mcp__commonly__|commonly_)/.test(name)) return 'commonly'; + if (/^mcp__[^_]/.test(name)) return 'connector'; + return 'native'; +}; diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js new file mode 100644 index 0000000000..6d59ff26f7 --- /dev/null +++ b/packages/seats/src/record/otel.js @@ -0,0 +1,238 @@ +/** + * Export run records as OTLP/JSON traces (an ExportTraceServiceRequest), so + * a tracing backend can show who let which agent do what beside its own + * spans. docs/record-v1.md §5 is the mapping; the gen_ai.* names follow the + * OpenTelemetry GenAI conventions, which are Development status upstream. + * + * Ids are derived, never random: the same entries export to the same bytes. + */ +import { createHash } from 'node:crypto'; +import { KINDS, RECORD_SCHEMA, RECORD_VERSION } from './format.js'; + +const SPAN_KIND_INTERNAL = 1; +const STATUS_UNSET = 0; +const STATUS_ERROR = 2; +const FAILED_OUTCOMES = new Set(['failed', 'stuck', 'cancelled']); +// Content stays out of the export, as upstream makes content capture opt-in. +const CONTENT_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']); + +const digest = (text, bytes) => { + const hex = createHash('sha256').update(text, 'utf8').digest('hex').slice(0, bytes * 2); + // An all-zero id is invalid in OTLP; vanishingly unlikely, cheap to exclude. + return /^0+$/.test(hex) ? digest(`${text}#`, bytes) : hex; +}; + +/** 16-byte trace id per task (task ids repeat across pods, so the pod is in it), else per run. */ +export const traceIdOf = (entry) => digest(entry.taskId + ? `seats.record/trace/task/${entry.podId}/${entry.taskId}` + : `seats.record/trace/run/${entry.runId}`, 16); +/** 8-byte span id for the span an entry opens; the run span uses seq 0. */ +export const spanIdOf = (runId, seq) => digest(`seats.record/span/${runId}/${seq}`, 8); + +/** RFC 3339 → Unix nanoseconds as a decimal string, keeping sub-millisecond digits. */ +const nanos = (ts) => { + const match = /^(.*T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z$/.exec(ts); + if (!match) return '0'; + return String(BigInt(Date.parse(`${match[1]}Z`)) * 1000000n + BigInt((match[2] || '').padEnd(9, '0'))); +}; + +const snake = (key) => key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); + +const anyValue = (value) => { + if (typeof value === 'string') return { stringValue: value }; + if (typeof value === 'boolean') return { boolValue: value }; + if (Number.isInteger(value)) return { intValue: String(value) }; + if (typeof value === 'number') return { doubleValue: value }; + if (Array.isArray(value)) return { arrayValue: { values: value.filter((v) => v !== null && v !== undefined).map(anyValue) } }; + return { + kvlistValue: { + values: Object.keys(value).filter((k) => value[k] !== null && value[k] !== undefined).sort() + .map((k) => ({ key: snake(k), value: anyValue(value[k]) })), + }, + }; +}; +const attr = (key, value) => ({ key, value: anyValue(value) }); +const attrs = (pairs) => pairs.filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => attr(key, value)); + +/** Nested data → `seats.` attributes, keys sorted for a stable export. */ +const flatten = (data, prefix = 'seats') => Object.keys(data || {}).sort().flatMap((key) => { + const value = data[key]; + const name = `${prefix}.${snake(key)}`; + if (value === null || value === undefined) return []; + if (typeof value === 'object' && !Array.isArray(value)) return flatten(value, name); + return [attr(name, value)]; +}); + +const eventOf = (entry) => ({ + timeUnixNano: nanos(entry.ts), + name: entry.kind, + attributes: [ + attr('seats.seq', entry.seq), + ...(entry.actor ? attrs([ + ['seats.actor.type', entry.actor.type], ['seats.actor.id', entry.actor.id], + ['seats.actor.owner', entry.actor.owner], ['seats.actor.name', entry.actor.name], + ['seats.actor.on_behalf_of.id', entry.actor.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor.onBehalfOf?.role], + ]) : []), + ...flatten(entry.data), + ], +}); + +const fail = (span, errorType) => { + span.status = { code: STATUS_ERROR, message: errorType }; + span.attributes.push(attr('error.type', errorType)); +}; + +/** One run's entries, in seq order → its spans. */ +const spansOfRun = (entries) => { + const head = entries[0]; + const traceId = traceIdOf(head); + const last = entries[entries.length - 1]; + const span = ({ seq, parent, name, attributes, start }) => ({ + traceId, + spanId: spanIdOf(head.runId, seq), + ...(parent ? { parentSpanId: parent.spanId } : {}), + name, + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: nanos(start), + endTimeUnixNano: null, + attributes, + events: [], + status: { code: STATUS_UNSET }, + }); + const run = span({ + seq: 0, + name: 'seats.run', + start: head.ts, + attributes: attrs([['seats.run.id', head.runId], ['seats.task.id', head.taskId], ['seats.pod.id', head.podId]]), + }); + const spans = [run]; + let attempt = null; // { span, number, tools: Map, usage, context, sessionId } + const cards = new Map(); // decisionCardId → the span its request sits on + const end = (s, ts) => { if (s && s.endTimeUnixNano === null) s.endTimeUnixNano = nanos(ts); }; + const closeAttempt = (ts) => { + if (!attempt) return; + for (const tool of attempt.tools.values()) end(tool, ts); + if (attempt.usage) { + attempt.span.attributes.push(...attrs([ + ['gen_ai.usage.input_tokens', attempt.usage.inputTokens], ['gen_ai.usage.output_tokens', attempt.usage.outputTokens], + ])); + } + attempt.span.attributes.push(...attrs([ + ['seats.usage.context_tokens', attempt.context.tokens], ['seats.usage.context_size', attempt.context.size], + ])); + // The session an attempt opened names the conversation when attempt.started did not. + if (attempt.sessionId && !attempt.span.attributes.some((a) => a.key === 'gen_ai.conversation.id')) { + attempt.span.attributes.push(attr('gen_ai.conversation.id', attempt.sessionId)); + } + end(attempt.span, ts); + attempt = null; + }; + const current = () => attempt?.span || run; + + for (const entry of entries) { + const { kind, data = {} } = entry; + if (!Object.hasOwn(KINDS, kind) || CONTENT_KINDS.has(kind)) continue; // unknown kinds are skipped + if (kind === 'attempt.started') { + closeAttempt(entry.ts); + const agentName = entry.actor?.name || entry.actor?.id; + const s = span({ + seq: entry.seq, + parent: run, + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'invoke_agent'], + ['gen_ai.agent.id', entry.actor?.id], ['gen_ai.agent.name', agentName], + ['gen_ai.conversation.id', data.sessionId], + ['seats.agent.owner', entry.actor?.owner], + ['seats.agent.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.agent.on_behalf_of.role', entry.actor?.onBehalfOf?.role], + ['seats.attempt.number', data.attempt], ['seats.adapter', data.adapter], + ['seats.cli.name', data.cli?.name], ['seats.cli.version', data.cli?.version], + ]), + }); + spans.push(s); + attempt = { + span: s, number: data.attempt, agentName, tools: new Map(), usage: null, context: { tokens: null, size: null }, sessionId: null, + }; + } else if (kind === 'attempt.settled' && attempt && attempt.number === data.attempt) { + attempt.span.attributes.push(attr('seats.attempt.outcome', data.outcome)); + if (FAILED_OUTCOMES.has(data.outcome)) fail(attempt.span, data.failure?.kind || data.outcome); + closeAttempt(entry.ts); + } else if (kind === 'run.settled') { + run.attributes.push(...attrs([['seats.run.outcome', data.outcome], ['seats.run.attempts', data.attempts]])); + if (FAILED_OUTCOMES.has(data.outcome)) { + const errorType = data.failure?.kind || data.outcome; + fail(run, errorType); + // The run's outcome is the last attempt's when nothing settled it. + if (attempt) fail(attempt.span, errorType); + } + closeAttempt(entry.ts); + end(run, entry.ts); + } else if (kind === 'tool.call') { + const toolName = data.name || data.title || 'tool'; + const s = span({ + seq: entry.seq, + parent: current(), + name: `execute_tool ${toolName}`, + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'execute_tool'], + ['gen_ai.tool.name', toolName], ['gen_ai.tool.call.id', data.toolCallId], + ['gen_ai.agent.name', attempt?.agentName], + ['seats.actor.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor?.onBehalfOf?.role], + ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], ['seats.tool.args_digest', data.argsDigest], + ]), + }); + spans.push(s); + if (attempt && data.toolCallId) attempt.tools.set(data.toolCallId, s); + } else if (kind === 'tool.update') { + const tool = attempt?.tools.get(data.toolCallId); + (tool || current()).events.push(eventOf(entry)); + if (tool && data.status === 'failed') fail(tool, 'tool_failed'); + if (tool && (data.status === 'completed' || data.status === 'failed')) end(tool, entry.ts); + } else if (kind === 'usage' && attempt) { + attempt.usage = data; + // Context fill comes from the stream's usage updates; a later entry without it keeps the last known. + if (Number.isInteger(data.contextTokens)) attempt.context.tokens = data.contextTokens; + if (Number.isInteger(data.contextSize)) attempt.context.size = data.contextSize; + attempt.span.events.push(eventOf(entry)); + } else if (kind === 'permission.requested' || kind === 'permission.decided') { + const target = attempt?.tools.get(data.toolCallId) || current(); + target.events.push(eventOf(entry)); + if (data.decisionCardId && !cards.has(data.decisionCardId)) cards.set(data.decisionCardId, target); + } else if (kind === 'session.opened') { + if (attempt && typeof data.sessionId === 'string') attempt.sessionId = data.sessionId; + current().events.push(eventOf(entry)); + } else if (kind === 'decision.ruled') { + // A ruling on a card that a request rode on joins that request; any other is an event where it happened. + (cards.get(data.decisionCardId) || current()).events.push(eventOf(entry)); + } else { + current().events.push(eventOf(entry)); + } + } + closeAttempt(last.ts); + end(run, last.ts); + return spans; +}; + +/** + * toOtlpJson(entries, { serviceName? }) → ExportTraceServiceRequest (OTLP/JSON) + * + * `entries` are record entries (readRun's output, or parsed lines) from one + * or more runs. Entries of another schema or a newer version are left out. + */ +export const toOtlpJson = (entries, { serviceName = 'seats' } = {}) => { + const runs = new Map(); + for (const entry of entries || []) { + if (!entry || entry.schema !== RECORD_SCHEMA || !(entry.v <= RECORD_VERSION)) continue; + if (!runs.has(entry.runId)) runs.set(entry.runId, []); + runs.get(entry.runId).push(entry); + } + const spans = [...runs.values()].flatMap((run) => spansOfRun([...run].sort((a, b) => a.seq - b.seq))); + return { + resourceSpans: [{ + resource: { attributes: [attr('service.name', serviceName)] }, + scopeSpans: [{ scope: { name: RECORD_SCHEMA, version: String(RECORD_VERSION) }, spans }], + }], + }; +}; diff --git a/packages/seats/src/runtime-observation.js b/packages/seats/src/runtime-observation.js new file mode 100644 index 0000000000..e9c7bfd48d --- /dev/null +++ b/packages/seats/src/runtime-observation.js @@ -0,0 +1,29 @@ +// Read only adapter-owned control arguments. Callers exclude prompt/stdin and +// provide every value-taking option, so a flag-shaped model/session value is +// never reinterpreted as a second option. Raw arguments never leave the runner. +export const controlsFromArguments = (args, { modelFlag, effortFlag, valueFlags = [], codex = false }) => { + const controls = { model: null, effort: null }; + const flags = new Set([modelFlag, effortFlag, ...valueFlags].filter(Boolean)); + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + if (!flags.has(flag)) continue; + const value = args[++index]; + if (flag === modelFlag) controls.model = value || null; + if (flag === effortFlag) controls.effort = value || null; + if (codex && flag === '-c' && value?.startsWith('model_reasoning_effort=')) { + try { controls.effort = JSON.parse(value.slice('model_reasoning_effort='.length)); } catch { /* unknown encoding */ } + } + } + return controls; +}; + +// Called at the child's successful OS spawn event, not when a requested +// configuration is saved or when a turn is merely queued. Never send argv. +// The child rides along as a second argument so a supervisor can read its +// output as a liveness signal and kill it; a reporter reads only the first. +export const observeSpawn = (child, onSpawn, controls) => { + if (typeof onSpawn !== 'function') return; + child.once('spawn', () => { + try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }, child); } catch { /* diagnostic only */ } + }); +}; diff --git a/packages/seats/src/seat-model.js b/packages/seats/src/seat-model.js new file mode 100644 index 0000000000..b672e394d2 --- /dev/null +++ b/packages/seats/src/seat-model.js @@ -0,0 +1,17 @@ +/** + * The model a seat runs on is a declared field (`environment.model`), never a + * CLI default the operator did not choose. Whether a missing model is an error + * is the host's policy, passed as `required`. + */ +export const requireSeatModel = (environment, adapterName, required = false) => { + const model = environment?.model; + if (typeof model !== 'string' || !model.trim()) { + if (!required) return undefined; + const error = new Error( + `${adapterName} requires environment.model; set the seat model with commonly agent config --model before running`, + ); + error.code = 'seat_model_missing'; + throw error; + } + return model.trim(); +}; diff --git a/packages/seats/src/supervisor.js b/packages/seats/src/supervisor.js new file mode 100644 index 0000000000..7bfd471b1a --- /dev/null +++ b/packages/seats/src/supervisor.js @@ -0,0 +1,308 @@ +/** + * The run supervisor: one prompt, one traced run, and a guarantee that the run + * settles. A turn that hangs is named `stuck` in the trace, cancelled, and if + * the adapter will not let go, abandoned and settled anyway. Nothing this + * module starts is allowed to sit in "running" forever. + * + * Liveness is read from progress signals, not from the reply: any trace event + * the adapter emits, and any byte on the stdout/stderr of a child the adapter + * reports through `ctx.onSpawn`. A run that is busy but quiet past the window + * is indistinguishable from a hung one, which is why the window is long. + * + * The adapter is told (ctx.ownProcessGroup) that it may start its child as the + * leader of a new process group; a child it marks as one is killed as a group, + * so the CLI's own children (MCP servers, a native binary behind a launcher) + * die with it. See process-group.js, which also kills those groups when the + * wrapper itself is signalled or exits. + */ +import { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; +import { redactKnownCredentials, scrubAdapterFailure, spawnCredentials } from './upstream-refusal.js'; +import { killProcessTree, trackSupervisedGroup, OWN_PROCESS_GROUP } from './process-group.js'; + +export const SUPERVISOR_DEFAULTS = Object.freeze({ + // Longer than the quiet stretches a healthy turn has: a long-context first + // token, or a tool (an install, a test suite) whose output the CLI buffers. + stallMs: 120_000, + // Covers an adapter's own SIGTERM -> 5 s -> SIGKILL ladder with room to spare. + cancelGraceMs: 10_000, + // The turn budget the run loop already enforces (COMMONLY_AGENT_RUN_TIMEOUT_MS's + // fallback), so supervising a turn does not shorten or lengthen it. + timeoutMs: 15 * 60 * 1000, +}); + +export const DEFAULT_RETRY_ON = Object.freeze(['stuck', 'exited', 'timeout']); + +// setTimeout overflows past this and fires immediately. +const MAX_TIMER_MS = 2_147_483_647; +// The supervisor's own events; an adapter writing them would forge the run's outcome. +const SUPERVISOR_EVENTS = new Set(['run.started', 'run.settled']); + +const windowOf = (value, fallback) => { + if (value === undefined || value === null) return fallback; + const number = Number(value); + if (number === 0 || number === Infinity) return null; // explicitly off + if (!Number.isFinite(number) || number < 0) throw new Error(`supervisor window must be a non-negative number, got ${value}`); + return Math.min(number, MAX_TIMER_MS); +}; + +const resolvePolicy = (policy = {}) => { + const retry = policy.retry || {}; + return { + stallMs: windowOf(policy.stallMs, SUPERVISOR_DEFAULTS.stallMs), + cancelGraceMs: windowOf(policy.cancelGraceMs, SUPERVISOR_DEFAULTS.cancelGraceMs) ?? 0, + timeoutMs: windowOf(policy.timeoutMs, SUPERVISOR_DEFAULTS.timeoutMs), + retryMax: Math.max(0, Math.floor(Number(retry.max) || 0)), + retryOn: Array.isArray(retry.on) ? retry.on : DEFAULT_RETRY_ON, + onStalled: typeof policy.onStalled === 'function' ? policy.onStalled : null, + }; +}; + +const isChild = (value) => !!value && typeof value.kill === 'function' && typeof value.once === 'function'; +const isAlive = (child) => child.exitCode === null && child.signalCode === null; + +// The retryable flag in FAILURE_KINDS outranks the caller's list: a refusal +// listed in `on` is still never re-sent. +const shouldRetry = (failure, attempt, settings) => { + const kind = failureKindOf(failure); + return attempt <= settings.retryMax + && !!kind && FAILURE_KINDS[kind].retryable + && settings.retryOn.includes(kind) + && !failure.adapterIgnoredCancel; +}; + +const outcomeOf = (failure) => { + if (!failure) return 'ok'; + const kind = failureKindOf(failure); + if (kind === 'stuck') return 'stuck'; + if (kind === 'cancelled') return 'cancelled'; + return 'failed'; +}; + +// An adapter that threw a bare Error broke the RunFailure rule; its text is +// still scrubbed before it reaches the trace, and it carries no kind rather +// than a guessed one. A RunFailure's detail keeps its shape but loses any +// credential this spawn was handed, whoever built it. +const failureData = (failure, credentials) => (failure instanceof RunFailure + ? { kind: failure.kind, detail: redactKnownCredentials(failure.detail, credentials) } + : { kind: null, detail: scrubAdapterFailure(String(failure?.message ?? failure ?? ''), { credentials }).detail }); + +const sessionIdOf = (failure) => { + const id = failure?.sessionId ?? failure?.newSessionId; + return typeof id === 'string' && id ? id : null; +}; + +const ignoredCancel = (reason, graceMs) => { + const failure = new RunFailure(reason.kind, `${reason.detail}; the adapter did not settle within ${graceMs}ms of cancellation`); + failure.adapterIgnoredCancel = true; + return failure; +}; + +const callerCancel = (signal) => (failureKindOf(signal.reason) === 'cancelled' + ? signal.reason : runFailure('cancelled', 'cancelled by the caller')); + +/** One attempt. Resolves { result } or { failure }; never rejects, never stays pending. */ +const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, external }) => new Promise((resolve) => { + const controller = new AbortController(); + const children = new Set(); + let live = true; + let abortedBy = null; + let lastSignal = Date.now(); + let stallTimer = null; + let wallTimer = null; + let graceTimer = null; + + const touch = () => { if (live) lastSignal = Date.now(); }; + + const killChildren = () => { + let killed = 0; + for (const child of children) { + if (!isAlive(child)) continue; + if (killProcessTree(child, 'SIGKILL')) killed += 1; + } + return killed; + }; + + const onExternal = () => cancel(callerCancel(external), { recovery: false }); + + const finish = (outcome) => { + if (!live) return; + live = false; + clearTimeout(stallTimer); + clearTimeout(wallTimer); + clearTimeout(graceTimer); + external?.removeEventListener?.('abort', onExternal); + // An adapter that honoured the abort can still leave its child behind. + if (abortedBy && !outcome.failure?.adapterIgnoredCancel && killChildren() > 0) { + run.emit('liveness.recovery', { action: 'kill', attempt }); + } + resolve(outcome); + }; + + function cancel(reason, { recovery = true } = {}) { + if (abortedBy || !live) return; + abortedBy = reason; + clearTimeout(stallTimer); + clearTimeout(wallTimer); + if (recovery) run.emit('liveness.recovery', { action: 'cancel', attempt }); + controller.abort(reason); + graceTimer = setTimeout(() => { + if (!live) return; + run.emit('liveness.recovery', { action: 'kill', attempt }); + killChildren(); + finish({ failure: ignoredCancel(reason, settings.cancelGraceMs) }); + }, settings.cancelGraceMs); + } + + const checkStall = () => { + if (!live || abortedBy) return; + const idleMs = Date.now() - lastSignal; + if (idleMs < settings.stallMs) { + stallTimer = setTimeout(checkStall, settings.stallMs - idleMs); + return; + } + const windowMs = settings.stallMs; + run.emit('liveness.stalled', { idleMs, windowMs }); + if (settings.onStalled) { + // A host's notifier failing must not keep the run from recovering. + try { + Promise.resolve(settings.onStalled({ runId: run.runId, adapter: adapter.name, attempt, idleMs, windowMs })).catch(() => {}); + } catch { /* notifier only */ } + } + cancel(runFailure('stuck', `no progress signal for ${idleMs}ms (window ${windowMs}ms)`)); + }; + + // Counts as progress only while this attempt is live, so a cancelled + // attempt's late output cannot keep the next one alive or reach the trace. + // The emit options (the entry's actor) pass through unchanged. `redact` is + // offered only when the run has one, so prepareRunEnv cannot hand its + // values to a no-op; it is not gated on `live`, since adding a value to + // scrub is never unsafe. + const trace = { + emit(type, data, options) { + if (!live || SUPERVISOR_EVENTS.has(type)) return null; + touch(); + return run.emit(type, data, options); + }, + ...(typeof run.redact === 'function' ? { redact: (values) => run.redact(values) } : {}), + }; + + // Wraps the caller's onSpawn: every argument reaches it unchanged. + // A group-leading child is tracked for the wrapper's exit even when it + // arrives after this attempt ended: it is outside the seat's own group, so + // nothing else would kill it. + const onSpawn = (...args) => { + for (const child of args.filter(isChild)) trackSupervisedGroup(child); + if (live) { + touch(); + for (const child of args.filter(isChild)) { + children.add(child); + child.stdout?.on?.('data', touch); + child.stderr?.on?.('data', touch); + } + } + return typeof ctx.onSpawn === 'function' ? ctx.onSpawn(...args) : undefined; + }; + + if (external?.aborted) { + abortedBy = callerCancel(external); + finish({ failure: abortedBy }); + return; + } + external?.addEventListener?.('abort', onExternal, { once: true }); + if (settings.stallMs) stallTimer = setTimeout(checkStall, settings.stallMs); + if (settings.timeoutMs) { + wallTimer = setTimeout(() => cancel(runFailure('timeout', `exceeded the ${settings.timeoutMs}ms wall-clock budget`)), settings.timeoutMs); + } + + let pending; + try { + pending = Promise.resolve(adapter.spawn(prompt, { + ...ctx, ...(sessionId ? { sessionId } : {}), trace, signal: controller.signal, onSpawn, [OWN_PROCESS_GROUP]: true, + })); + } catch (error) { + pending = Promise.reject(error); + } + // Once the supervisor has aborted, the abort's reason is the outcome, even + // if the adapter answers inside the grace window: the trace already says why. + pending.then( + (result) => finish(abortedBy ? { failure: abortedBy } : { result }), + (error) => finish({ failure: abortedBy || error }), + ); +}); + +/** + * Run one prompt under liveness supervision. + * + * @param {object} args + * @param {import('./contract.js').Adapter} args.adapter + * @param {string} args.prompt + * @param {object} [args.ctx] passed to adapter.spawn; `trace`, `signal` and `onSpawn` are wrapped, + * and `ownProcessGroup` is set to true. + * `ctx.signal`, when given, cancels the run (outcome `cancelled`). + * @param {ReturnType} args.store + * @param {object} [args.policy] + * @param {number} [args.policy.stallMs] no-progress window; 0 turns it off + * @param {number} [args.policy.cancelGraceMs] how long a cancelled adapter has to settle + * @param {number} [args.policy.timeoutMs] wall-clock budget per attempt; 0 turns it off + * @param {{max: number, on?: string[]}} [args.policy.retry] off unless max > 0 + * @param {(info: {runId, adapter, attempt, idleMs, windowMs}) => void} [args.policy.onStalled] + * @param {string} [args.runId] + * @param {string} [args.label] + * @param {string} [args.taskId] the board task this run works; needs `podId` + * @param {string} [args.podId] + * @param {object} [args.actor] the run's own party, {type, id, owner?, onBehalfOf?}; + * entries the run writes without naming an actor (run.started, tool calls, + * permission requests) record this one instead of null + * @returns {Promise<{ runId: string, outcome: string, attempts: number, result?: object, failure?: Error }>} + */ +export const superviseRun = async ({ + adapter, prompt, ctx = {}, store, policy = {}, runId, label, taskId, podId, actor, +} = {}) => { + if (!adapter || typeof adapter.spawn !== 'function') throw new Error('superviseRun needs an adapter with spawn()'); + if (!store || typeof store.startRun !== 'function') throw new Error('superviseRun needs a trace store'); + const settings = resolvePolicy(policy); + const run = store.startRun({ + runId, adapter: adapter.name, label, taskId, podId, actor, + }); + // The runtime token rides in ctx, not through the valet, so the store would + // not otherwise know to scrub it from agent text. + if (typeof run.redact === 'function' && typeof ctx.runtimeToken === 'string' && ctx.runtimeToken) { + run.redact([ctx.runtimeToken]); + } + const startedAt = Date.now(); + const external = ctx.signal; + let attempt = 0; + let sessionId = ctx.sessionId || null; + let last = { failure: null }; + + try { + for (;;) { + attempt += 1; + run.emit('run.started', { adapter: adapter.name, ...(sessionId ? { sessionId } : {}), attempt }); + last = await runAttempt({ adapter, prompt, ctx, run, settings, attempt, sessionId, external }); + if (!last.failure || external?.aborted || !shouldRetry(last.failure, attempt, settings)) break; + sessionId = sessionIdOf(last.failure) || sessionId; + run.emit('liveness.recovery', { action: 'retry', attempt: attempt + 1 }); + } + } catch (error) { + // The supervisor's own fault (a trace write failing, say) still settles the run. + last = { failure: error }; + } + + const outcome = outcomeOf(last.failure); + const credentials = spawnCredentials(ctx, [adapter.providerKeyEnv].filter(Boolean)); + try { + run.emit('run.settled', { + outcome, + ...(last.failure ? { failure: failureData(last.failure, credentials) } : {}), + durationMs: Date.now() - startedAt, + attempts: attempt, + }); + } catch { /* the outcome is still returned */ } + return { + runId: run.runId, + outcome, + attempts: attempt, + ...(last.failure ? { failure: last.failure } : { result: last.result }), + }; +}; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js new file mode 100644 index 0000000000..229a7ca7f9 --- /dev/null +++ b/packages/seats/src/trace/events.js @@ -0,0 +1,45 @@ +/** + * Trace event vocabulary. Names follow ACP's session/update kinds where one + * exists, so an ACP agent's stream maps across without a translation table. + * + * Event: { runId, seq, ts, type, data }, plus the record envelope (kind, v, + * actor, prev, taskId?, podId?) when read back from the store; the on-disk + * format is docs/record-v1.md. `seq` is per run, starting at 1; `ts` is + * an ISO string. Events never carry prompt text, credentials or + * environment values — only names, ids, sizes and the agent's own output. + */ +export const EVENT_TYPES = Object.freeze([ + 'run.started', // { adapter, cli: {name, version}?, sessionId?, attempt } + 'run.settled', // { outcome: 'ok'|'failed'|'stuck'|'cancelled', failure?: {kind, detail}, durationMs, attempts } + 'turn.prompt_sent', // { chars } + 'agent.message_chunk', // { text } + 'agent.thought_chunk', // { chars } thoughts are counted, not stored + 'agent.retry', // { state: 'scheduled'|'started', attempt?, nextRetryAt?, status?, errorType? } the agent retrying its provider; never the provider's text + 'tool.call', // { toolCallId, title, kind?, status?, name?, surface?, argsDigest? } + 'tool.update', // { toolCallId, status?, title? } + 'plan', // { entries: [{content, status}] } + 'permission.requested', // { toolCallId?, options: [{optionId, kind}], question, askedOf: {id, role}, grantId?, decisionCardId?, argsDigest? } + 'permission.decided', // { outcome: 'selected'|'cancelled', optionId?, kind?, toolCallId?, rule?: {list, pattern}, approval? } + 'usage', // { inputTokens?, outputTokens?, totalTokens?, cost? } null-not-zero when unknown + 'liveness.stalled', // { idleMs, windowMs } + 'liveness.recovery', // { action: 'cancel'|'kill'|'retry', attempt } + 'credential.granted', // { names: [...] } names only, never values + 'credential.denied', // { name, reason?: 'not_granted'|'unresolved'|'too_short' } + 'process.exited', // { code, signal } + // Record kinds with no earlier trace name (docs/record-v1.md has the full data). + 'task.claimed', // { leaseUntil? } mirrors the board's claim; the board stays the source of truth + 'task.released', // { reason? } + 'task.done', // { doneBy? } written for the board; ends grants bound to the task + 'grant.issued', // { grantId, parentGrantId?, scope: {connectionId?, tools: [names], writeMode?, budget?, audience}, target?, issuedBy?, expiresAt?, binding? } + 'grant.revoked', // { grantIds: [...], reason: 'task.done'|'expired'|'cascade'|'manual' } + 'attempt.settled', // { attempt, outcome, failure?: {kind, detail} } + 'decision.ruled', // { decisionCardId, value, decidedBy, decidedByRole } a card ruling not tied to a parked tool call + 'result.reported', // { status: 'done'|'partial'|'blocked'|'failed', refs?: [{type, id}] } +]); + +const KNOWN = new Set(EVENT_TYPES); +export const isEventType = (type) => KNOWN.has(type); + +/** The trace hook an adapter receives as ctx.trace. Absent hook → no-op. */ +export const NULL_TRACE = Object.freeze({ emit() {} }); +export const traceOf = (ctx) => (ctx && ctx.trace && typeof ctx.trace.emit === 'function' ? ctx.trace : NULL_TRACE); diff --git a/packages/seats/src/trace/store.js b/packages/seats/src/trace/store.js new file mode 100644 index 0000000000..5049f8d82a --- /dev/null +++ b/packages/seats/src/trace/store.js @@ -0,0 +1,276 @@ +/** + * Append-only trace store on the local filesystem. + * + * /runs/.jsonl the run record (docs/record-v1.md): one + * canonical, hash-chained entry per line + * /index.jsonl one line per started run, one per settled run; + * a store convenience, not part of the format + * + * Files are 0600 and directories 0700: a trace holds agent output. + */ +import { randomUUID } from 'node:crypto'; +import { appendFileSync, mkdirSync, readFileSync, existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + appendEntry, hashLine, kindOfType, readRecord, traceTypeOf, validateActor, verifyRecord, BOARD_ACTOR, +} from '../record/format.js'; + +const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; +const readJsonl = (path) => (existsSync(path) + ? readFileSync(path, 'utf8').split('\n').filter(Boolean).flatMap((line) => { + try { return [JSON.parse(line)]; } catch { return []; } + }) + : []); + +/** + * Known credential values never reach a line, even inside agent text. Every + * registered value is replaced, whatever its length: the eight-character + * floor in redactKnownCredentials belongs to the refusal-text reader, not to + * the record. `secrets` is kept longest first, so a value that contains a + * shorter one is replaced whole rather than in pieces. + */ +const scrubText = (text, secrets) => { + let out = text; + for (const value of secrets) out = out.split(value).join('[redacted]'); + return out; +}; + +const scrub = (value, secrets) => { + if (secrets.length === 0) return value; + if (typeof value === 'string') return scrubText(value, secrets); + if (Array.isArray(value)) return value.map((item) => scrub(item, secrets)); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrub(item, secrets)])); + } + return value; +}; + +/** + * Streamed agent text arrives in deltas, so a value can be split across two or + * more of them and match none on its own. Each stream (kind + actor) is + * scrubbed over a rolling window instead: the held tail is joined to the next + * delta, the join is scrubbed, and the longest suffix of the result that + * could still begin a known value is held back again. What is written is the + * rest. + * + * Why that is enough: an occurrence that started inside written text and ran + * past its end would make the written part plus the held tail a proper prefix + * of that value, a longer such suffix than the one held. So no value spans a + * written boundary, and each written piece is itself scrubbed. The tail is + * shorter than the longest value, and only as long as an actual prefix match. + */ +const STREAMED_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']); +// The held text is written out before these, so it stays in its own attempt +// and is never lost at settle, whichever path (ok, failure, cancel, kill) got there. +const FLUSH_BEFORE = new Set(['attempt.started', 'attempt.settled', 'run.settled']); + +const heldLength = (text, secrets) => { + const longest = secrets.length ? secrets[0].length : 0; + for (let start = Math.max(0, text.length - longest + 1); start < text.length; start += 1) { + const tail = text.slice(start); + if (secrets.some((value) => value.length > tail.length && value.startsWith(tail))) return text.length - start; + } + return 0; +}; + +/** What readRun has always returned, plus the envelope: `type` is the trace name. */ +const asEvent = (entry) => (entry.kind ? { ...entry, type: traceTypeOf(entry.kind) } : entry); + +export const createTraceStore = ({ root, now = () => new Date() } = {}) => { + if (!root) throw new Error('createTraceStore needs a root directory'); + const runsDir = join(root, 'runs'); + mkdirSync(runsDir, { recursive: true, mode: 0o700 }); + const append = (path, value) => appendFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o600 }); + const indexPath = join(root, 'index.jsonl'); + + /** + * startRun({ runId?, adapter?, label?, taskId?, podId?, actor? }) + * → { runId, emit, redact, revokeGrant, expireGrants, openGrants, settled } + * + * `taskId` is the board's task id verbatim and needs `podId`, since task + * ids repeat across pods. `actor` is the run's own party, normally + * {type: 'agent', id: , owner: }. + */ + const startRun = (meta = {}) => { + const runId = meta.runId || randomUUID(); + if (!RUN_ID.test(runId)) throw new Error('runId must be 6-64 characters of [A-Za-z0-9_-]'); + const taskId = meta.taskId || undefined; + const podId = meta.podId || undefined; + if (taskId && !podId) throw new Error('a taskId needs a podId: task ids repeat across pods'); + const runActor = meta.actor === undefined ? null : validateActor(meta.actor); + const path = join(runsDir, `${runId}.jsonl`); + try { + writeFileSync(path, '', { mode: 0o600, flag: 'wx' }); + } catch (error) { + if (error.code === 'EEXIST') throw new Error(`run ${runId} already exists`); + throw error; + } + let last = null; // the previous line, which the next one hashes + let seq = 0; + let settled = false; + let broken = false; + const secrets = []; // longest first; see scrubText + const streams = new Map(); // stream key → { kind, actor, held } + const grants = new Map(); // open grants: grantId → { parentGrantId, binding, expiresAt } + const startedAt = now(); + + const writeLine = (kind, data, actor) => { + const line = appendEntry(last, { + runId, taskId, podId, ts: now().toISOString(), kind, data: scrub(data, secrets), actor, + }, { runActor }); + try { + appendFileSync(path, `${line}\n`, { mode: 0o600 }); + } catch (error) { + // Some bytes may have landed; a chain cannot continue past an unknown tail. + broken = true; + throw error; + } + last = line; + seq += 1; + const entry = JSON.parse(line); + if (kind === 'grant.issued') { + grants.set(entry.data.grantId, { + parentGrantId: entry.data.parentGrantId ?? null, binding: entry.data.binding || null, expiresAt: entry.data.expiresAt || null, + }); + } + if (kind === 'grant.revoked') for (const id of entry.data.grantIds) grants.delete(id); + return asEvent(entry); + }; + + /** A streamed delta's text, less a tail that could begin a known value. */ + const windowed = (kind, data, actor) => { + if (!STREAMED_KINDS.has(kind) || typeof data?.text !== 'string') return data; + const key = JSON.stringify([kind, actor ?? null]); + const joined = scrubText(`${streams.get(key)?.held ?? ''}${data.text}`, secrets); + const cut = joined.length - heldLength(joined, secrets); + if (cut < joined.length) streams.set(key, { kind, actor, held: joined.slice(cut) }); + else streams.delete(key); + return { ...data, text: joined.slice(0, cut) }; + }; + + const write = (kind, data, actor) => writeLine(kind, windowed(kind, data, actor), actor); + + /** + * The stream has ended, so each held tail can no longer complete a value. + * A tail is held only while it is a prefix of a known value, so it is part + * of a credential: write the marker in its place, never the prefix. + */ + const flushStreams = () => { + for (const [key, { kind, actor }] of streams) { + streams.delete(key); + writeLine(kind, { text: '[redacted]' }, actor); + } + }; + + const revoke = (grantIds, reason, actor) => (grantIds.length ? write('grant.revoked', { grantIds, reason }, actor) : null); + + const emit = (type, data = {}, options = {}) => { + const kind = kindOfType(type); + if (!kind) throw new Error(`unknown trace event type: ${type}`); + if (settled) return null; + if (broken) throw new Error(`run ${runId}: the record is broken after a failed write`); + if (FLUSH_BEFORE.has(kind)) flushStreams(); + const event = write(kind, data, options?.actor); + if (kind === 'task.done') { + // The board closed the task: every grant bound to it ends, in one write. + const bound = [...grants].filter(([, g]) => g.binding?.endsOn === 'task.done').map(([id]) => id); + revoke(bound, 'task.done', BOARD_ACTOR); + } + if (kind === 'run.settled') { + settled = true; + append(indexPath, { + runId, + settledAt: event.ts, + outcome: event.data.outcome, + failure: event.data.failure || null, + durationMs: event.data.durationMs ?? null, + entries: seq, + head: hashLine(last), + }); + } + return event; + }; + + /** Revoke a grant and every descendant this run knows of, in one entry. */ + const revokeGrant = (grantId, { actor } = {}) => { + if (settled || !grants.has(grantId)) return null; + const lineage = [grantId]; + for (let i = 0; i < lineage.length; i += 1) { + for (const [id, g] of grants) if (g.parentGrantId === lineage[i] && !lineage.includes(id)) lineage.push(id); + } + return revoke(lineage, lineage.length > 1 ? 'cascade' : 'manual', actor); + }; + + /** Record the end of every open grant whose expiresAt has passed. */ + const expireGrants = (at = now()) => { + if (settled) return null; + const cutoff = at.getTime(); + return revoke([...grants].filter(([, g]) => g.expiresAt && Date.parse(g.expiresAt) <= cutoff).map(([id]) => id), 'expired'); + }; + + /** Every non-empty string is scrubbed from here on, whatever its length. */ + const redact = (values = []) => { + for (const value of values) if (typeof value === 'string' && value && !secrets.includes(value)) secrets.push(value); + secrets.sort((a, b) => b.length - a.length); + }; + + append(indexPath, { + runId, + startedAt: startedAt.toISOString(), + adapter: meta.adapter || null, + label: meta.label || null, + ...(taskId ? { taskId, podId } : podId ? { podId } : {}), + ...(runActor ? { actor: runActor } : {}), + }); + return { + runId, + taskId: taskId ?? null, + podId: podId ?? null, + emit, + redact, + revokeGrant, + expireGrants, + openGrants: () => [...grants.keys()], + get settled() { return settled; }, + }; + }; + + const listRuns = ({ limit = 50 } = {}) => { + const byId = new Map(); + for (const line of readJsonl(indexPath)) { + const row = byId.get(line.runId) || { runId: line.runId, outcome: 'running' }; + Object.assign(row, line); + byId.set(line.runId, row); + } + return [...byId.values()].sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt))).slice(0, limit); + }; + + const runPath = (runId) => (RUN_ID.test(String(runId)) ? join(runsDir, `${runId}.jsonl`) : null); + + const readRun = (runId) => { + const path = runPath(runId); + return path && existsSync(path) ? readRecord(readFileSync(path, 'utf8')).map(asEvent) : null; + }; + + /** + * verifyRun(runId) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } | null + * + * The chain check from verifyRecord, plus the last line against the head the + * index recorded at settle (head_mismatch, count_mismatch). The index sits + * beside the record, so that check catches accidents, not an editor. + */ + const verifyRun = (runId) => { + const path = runPath(runId); + if (!path || !existsSync(path)) return null; + const result = verifyRecord(readFileSync(path, 'utf8')); + const sealed = readJsonl(indexPath).filter((row) => row.runId === runId && row.head).pop(); + if (sealed) { + if (sealed.entries !== result.entries) result.errors.push({ seq: sealed.entries, reason: 'count_mismatch', detail: `index ${sealed.entries}, file ${result.entries}` }); + else if (sealed.head !== result.head) result.errors.push({ seq: sealed.entries, reason: 'head_mismatch' }); + result.ok = result.errors.length === 0; + } + return result; + }; + + return { root, startRun, listRuns, readRun, verifyRun }; +}; diff --git a/packages/seats/src/upstream-refusal.js b/packages/seats/src/upstream-refusal.js new file mode 100644 index 0000000000..12f7f30cca --- /dev/null +++ b/packages/seats/src/upstream-refusal.js @@ -0,0 +1,362 @@ +/** + * TASK-096 — name a refused model route, instead of reporting the silence it + * causes. + * + * Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape + * that made a seat look broken for two and a half days): pi runs its own retry + * ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and + * EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only + * `type: "error"` events — of which this produces none — so the wrapper logged + * `no wrapper-post (empty output)` and the cause sat unread in the stream. + * + * Two rules govern the text that reaches a seat log: + * + * 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes + * only the shapes we thought of, and a 401 can echo the key back. So a body + * contributes one NAMED field or nothing at all. + * 2. Exact-match refusal for credentials. If the kept text contains a value + * this spawn was handed, the text is dropped — a known value, not a guessed + * shape (Wren 70877). + * + * Two limits follow from rule 2. They are the accepted cost of refusing guessed + * shapes, named here so they are not rediscovered as defects later (Vera 71082): + * + * a. The match is exact and against THIS spawn's values only, so a credential + * that arrives transformed — base64 inside a proxy error, percent-encoded in + * a URL echo — does not match and the text is kept. Matching transformed + * shapes instead would reintroduce the class rule 1 exists to avoid. + * b. The keep-list can still surface someone ELSE's secret. A third party's key + * relayed in an upstream error is not detectable here, because the one + * inventory this module has is the set of values it was itself handed. + * + * Scope: this reader was wired into the pi adapter only. TASK-103 extended it + * to codex and claude, which are NOT silent the way pi was: the claude adapter + * rejects on a non-zero exit and deliberately reports stdout with it, because a + * `-p` run writes quota conditions there; the codex adapter rejects on + * `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text + * they raised was the raw tail, unscrubbed by the rules above, and carried no + * HTTP status for the run loop to classify on. `scrubAdapterFailure` below is + * the shared reader they now both go through, so the two rules cannot drift + * apart between three adapters. + */ + +/** The kept detail is truncated, ellipsis included, to this many characters. */ +export const MAX_REFUSAL_DETAIL = 256; + +/** + * The status an adapter prefixes to its error text: `"429: {…}"` -> `429`. + * Null for anything that is not a 4xx/5xx, so a message that merely starts with + * a number ("4040 tokens over budget") is not read as a status. + * + * The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1 + * emits `"429: {"message":…}"` for a JSON body and `"502 …"` for an + * HTML one. Both forms are matched here; a parser that required the colon would + * have kept `502 …` as the whole "body" and then dropped it as non-JSON, + * reporting the status alone with no sign that the body was never read. + */ +const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/; + +export const statusFromError = (value) => { + const match = STATUS_PREFIX.exec(String(value ?? '')); + if (!match) return null; + const status = Number(match[1]); + return status >= 400 && status <= 599 ? status : null; +}; + +/** + * The body the adapter carried after its status prefix, or the whole text when + * there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`; + * `"502 "` -> ``. + */ +export const bodyFromError = (value) => { + const text = String(value ?? ''); + if (statusFromError(text) === null) return text.trim(); + return text.replace(STATUS_PREFIX, '').trim(); +}; + +/** The shortest value the refusal-text reader treats as a credential. */ +export const MIN_CREDENTIAL_LENGTH = 8; + +/** + * Replace credential values this spawn was handed with a marker. + * + * This is the exact-match half, for text that is kept for its whole shape rather + * than filtered to one field — an adapter's stderr tail, say, whose diagnostic + * value is in the lines around the secret. Values shorter than eight characters + * are ignored: redacting a one-character "credential" would corrupt ordinary + * prose, and no issued credential is that short. + * + * The floor belongs to this reader only. The trace store scrubs every value + * registered with it, whatever its length, and the valet refuses to grant a + * credential shorter than this (credential_denied), so no granted value is + * ever below the floor. + */ +export const redactKnownCredentials = (text, credentials = []) => { + let out = String(text ?? ''); + for (const value of credentials) { + if (typeof value !== 'string' || value.length < MIN_CREDENTIAL_LENGTH) continue; + out = out.split(value).join('[redacted]'); + } + return out; +}; + +/** + * The keep-list. Given the raw body an adapter captured, keep at most one named + * field of it and only when that field is a string: + * + * `error.message` first (the OpenAI-shaped error), then a top-level `message` + * (which is the shape LiteLLM's budget refusal actually uses — the motivating + * case; a keep-list that kept only `error.message` would name the status and + * still hide the budget line that started this row). + * + * Everything else — HTML, a non-JSON body, a JSON body with neither field, a + * body echoing a known credential — contributes nothing. Returns null rather + * than a partial guess. + */ +export const keptRefusalDetail = (raw, { credentials = [] } = {}) => { + let parsed; + try { + parsed = JSON.parse(String(raw ?? '')); + } catch { + return null; + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + + const candidate = parsed?.error?.message ?? parsed?.message; + if (typeof candidate !== 'string') return null; + + const text = candidate.replace(/\s+/g, ' ').trim(); + if (!text) return null; + + const redacted = redactKnownCredentials(text, credentials); + // `[redacted]` appearing is the signal that the body echoed something this + // spawn was handed. The status is still worth keeping; the text is not. + if (redacted !== text) return null; + + return text.length > MAX_REFUSAL_DETAIL + ? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…` + : text; +}; + +/** + * Read a pi JSON stream and return the refusal it ended on, or null. + * + * `auto_retry_end` is the terminal word — it names the status and the body after + * the ladder has finished — so it wins over the per-attempt `message_end` + * events that precede it. A run that refused on its first attempt and never + * retried still reports through `message_end`. + */ +export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => { + let status = null; + let raw = null; + + for (const line of String(stdout ?? '').split('\n')) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + + if (event?.type === 'auto_retry_end' && event.success === false) { + const candidate = event.finalError ?? event.errorMessage; + const attemptStatus = statusFromError(candidate); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(candidate); + } + continue; + } + + const message = event?.message; + if ( + event?.type === 'message_end' + && message?.role === 'assistant' + && message.stopReason === 'error' + && status === null + ) { + const attemptStatus = statusFromError(message.errorMessage); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(message.errorMessage); + } + } + } + + if (status === null) return null; + return { status, detail: keptRefusalDetail(raw, { credentials }) }; +}; + +/** + * Is this text STRUCTURED upstream data — a JSON object or an array? The gate for + * the whole-tail fallback below, deliberately NOT `kept === null`, which is true + * for five different reasons (Vera 71350). + * + * An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is + * structured data with fields this code does not know, which is exactly what rule + * 1 exists for, and treating it as prose shipped the whole body under exact-match + * alone. A JSON *string* is the opposite case and does NOT count — `"Too many + * requests"` is a whole-shape text with no fields to reduce, so it keeps its + * wording like any other prose. + */ +const isStructuredBody = (raw) => { + try { + const parsed = JSON.parse(String(raw ?? '')); + return parsed !== null && typeof parsed === 'object'; + } catch { + return false; + } +}; + +/** + * A 3-digit number is read as a status only when the tail NAMES it as one: + * `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | ` + * segment followed by a colon, which is the shape claude's `stderr | stdout` + * join produces for a body that leads with its status. + * + * A bare `\b(\d{3})\b` scan is not enough, and the false positives are not + * hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds` + * all read as 500, and `classifySpawnFailure` tests the status FIELD before the + * message — so a count misread as a status stands the fleet down on a refusal + * that never happened, which is worse than no status at all (Vera 71352). + */ +const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi; +// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is, +// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED +// line: the `1` in `/1.1` is a +// WORD character, so `HTTP` and `429` are not within four non-word characters of +// each other. It read as `status: null` — an unclassified runtime failure on the +// single most recognisable shape, which is the misclassification this row exists +// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap +// would start reading `/v1/models 500` as a status. +const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi; +const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g; + +/** The first 4xx/5xx one of those shapes names, or null. */ +const scanStatuses = (text, pattern) => { + for (const match of String(text ?? '').matchAll(pattern)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) return value; + } + return null; +}; + +/** + * Find the status AND the body boundary, because they are the same question. + * + * Anchoring the status to a ` | ` segment start is not enough on its own: the + * body has to be taken from just past that status too, or a mid-tail read + * isolates the status and then hands the WHOLE joined tail to the body rules, + * which no longer look like a JSON body and so fall through to "keep it all". + * That is the difference between dropping a credential-echoing body and shipping + * the part of it that exact-match cannot see. + * + * `STATUS_COLON` is the only shape that moves the boundary, because it is the + * same shape as the anchored prefix (`429: {body}`), just found at a segment + * boundary — claude's `stderr | stdout` join. A status that appears inside a + * body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status + * (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body. + */ +const readStatusAndBody = (text) => { + const raw = String(text ?? ''); + const prefixed = statusFromError(raw); + if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) }; + // Only the colon shape moves the boundary — see the note above. A status LINE + // names the status but its reason phrase is prose, so the body stays the whole + // text and the exact-match rule protects it. + for (const match of raw.matchAll(STATUS_COLON)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) { + return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') }; + } + } + const named = scanStatuses(raw, STATUS_NAMED); + return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw }; +}; + +/** + * The shared reader for an adapter's own failure tail (TASK-103). + * + * The two rules above are about a body an adapter captured, and they apply here + * too — but the tail is a different shape from pi's refusal, and the difference + * decides how each rule lands: + * + * - The tail is often NOT structured. Claude's `-p` mode writes its terminal + * condition as prose (`Claude usage limit reached. Your limit will reset at + * 11:40pm.`), and that line is the whole diagnostic value of reporting + * stdout at all — 361 consecutive failures on 2026-08-03 carried no reason + * because of it. So the keep-list applies only when the tail parses as a + * body; when it does not, the text is KEPT as a whole shape and the + * exact-match rule is what protects it (`redactKnownCredentials`, which + * substitutes rather than drops, precisely because a whole-shape text has no + * second field to fall back to). + * + * - A body that IS structured (a JSON object or array) is reduced to one named + * field, and a body whose field echoed a value this spawn was handed is + * DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the + * whole tail is gated on the SHAPE (is it structured at all), never on + * "the keep-list returned null": that null has + * four other causes, and one of them is the credential echo this line exists + * to drop. Gating on `kept === null` would emit the whole body for that case, + * which is precisely what rule 2 forbids — and exact-match cannot see a + * transformed shape (limit (a) in the header), so a key that arrived + * base64'd or percent-encoded would ship (Vera 71350/71351). + * + * Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null. + * The caller decides where the status goes; `classifySpawnFailure` reads it off + * `error.status` or out of the message text, so both a leading `429: …` prefix + * and an attached field classify the same way. `detail` is `''` when nothing of + * the body may be shown — the status is then the whole diagnostic. + */ +export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => { + const raw = String(text ?? '').trim(); + const { status, body } = readStatusAndBody(raw); + const kept = keptRefusalDetail(body, { credentials }); + const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials)); + return { status, detail: detail.slice(0, limit) }; +}; + +/** + * The Error an adapter raises, built from its own failure tail. + * + * `status` is attached as a field, not only written into the text: the run + * loop's classifier tests `error.status === 429` before it looks at the + * message, and a refusal that is only readable by regex is one wording away + * from the misclassification that cost an hour on 2026-08-18 (see + * spawn-retry.js). The message still names the status so a human log says the + * same thing as the classifier's decision. + */ +export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => { + const { status, detail } = scrubAdapterFailure(text, { credentials, limit }); + const code = exitCode === null ? '' : ` exited with code ${exitCode}`; + const named = status === null ? '' : ` (upstream ${status})`; + const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`); + if (status !== null) error.status = status; + return error; +}; + +/** + * The seat-log clause. The status is always present; the detail only when the + * keep-list kept one. `grep 'upstream refused'` finds every refusal, which is + * the property the old `no wrapper-post (empty output)` line did not have. + */ +export const describeUpstreamRefusal = (refusal) => { + if (!refusal || typeof refusal.status !== 'number') return null; + return refusal.detail + ? `upstream refused ${refusal.status}: ${refusal.detail}` + : `upstream refused ${refusal.status}`; +}; + +/** + * The credential values this spawn was handed, for the exact-match check in + * `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the + * caller supplies them because only the adapter knows which key its route uses. + */ +export const spawnCredentials = (ctx = {}, envNames = []) => { + const env = ctx.env || process.env || {}; + return [ + ctx.runtimeToken, + ...envNames.map((name) => env[name]), + ].filter((value) => typeof value === 'string' && value.length >= MIN_CREDENTIAL_LENGTH); +}; diff --git a/packages/seats/src/valet/env.js b/packages/seats/src/valet/env.js new file mode 100644 index 0000000000..fe37dfefbe --- /dev/null +++ b/packages/seats/src/valet/env.js @@ -0,0 +1,164 @@ +/** + * Build the env and HOME one run executes in. The run starts from nothing the + * user's login can be found through: a short allowlist of locale/terminal + * variables, a fresh HOME whose XDG dirs sit inside it, and only the + * credentials its grant names. + * + * What this is NOT: a filesystem sandbox. A process that already knows an + * absolute path (/Users//.claude) can still open it, and macOS keychain + * items are reached through the user's security session, not through HOME. + * This removes the paths a CLI discovers its login by; an OS sandbox + * (Seatbelt, bwrap) is what stops a process that goes looking. + */ +import { randomUUID } from 'node:crypto'; +import { mkdtemp, mkdir, chmod, readFile, rm } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { delimiter, join, resolve, sep } from 'node:path'; +import { traceOf } from '../trace/events.js'; +import { runFailure } from '../failures.js'; +import { MIN_CREDENTIAL_LENGTH } from '../upstream-refusal.js'; +import { validateGrant } from './grant.js'; + +// Same spirit as the goose adapter's SAFE_ENV: locale, terminal and TLS trust +// only. USER/LOGNAME are left out on purpose; they name the account whose +// login this boundary exists to hide. +export const BASE_ENV_ALLOWLIST = /^(?:PATH|LANG|LC_[A-Z0-9_]+|TERM|TZ|SSL_CERT_FILE|SSL_CERT_DIR|NO_COLOR|CI)$/; + +const isInside = (path, dir) => { + const a = resolve(path); + const b = resolve(dir); + return a === b || a.startsWith(b.endsWith(sep) ? b : b + sep); +}; + +/** + * PATH entries under the real home both name it and often hold shims that + * read the user's login (~/.local/bin/claude). Adapters run their CLI by + * absolute path, so the run loses nothing it needs. + */ +const scrubPath = (value, realHomes) => value.split(delimiter) + .filter((entry) => entry && !realHomes.some((home) => isInside(entry, home))) + .join(delimiter); + +const pickBaseEnv = (baseEnv, realHomes) => { + const env = {}; + for (const [name, value] of Object.entries(baseEnv)) { + if (typeof value === 'string' && BASE_ENV_ALLOWLIST.test(name)) env[name] = value; + } + if (env.PATH !== undefined) env.PATH = scrubPath(env.PATH, realHomes); + return env; +}; + +const makeHome = async (root) => { + const home = await mkdtemp(join(root || tmpdir(), 'seat-home-')); + // Conventional layout, so a CLI that ignores XDG_* and defaults to + // ~/.config still lands inside the throwaway HOME. + const dirs = { + XDG_CONFIG_HOME: join(home, '.config'), + XDG_DATA_HOME: join(home, '.local', 'share'), + XDG_STATE_HOME: join(home, '.local', 'state'), + XDG_CACHE_HOME: join(home, '.cache'), + TMPDIR: join(home, 'tmp'), + }; + await chmod(home, 0o700); + for (const dir of Object.values(dirs)) await mkdir(dir, { recursive: true, mode: 0o700 }); + // mkdir's mode is masked by umask; set it explicitly on every level made. + for (const dir of [join(home, '.local'), ...Object.values(dirs)]) await chmod(dir, 0o700); + return { home, dirs }; +}; + +/** A value that cannot ride an env var is the same as no value. */ +const usable = (value) => typeof value === 'string' && value.length > 0 && !value.includes('\0'); + +const resolveOne = async (spec, { baseEnv, resolveSecret }) => { + if (spec.from === 'env') return baseEnv[spec.ref || spec.name]; + if (spec.from === 'file') return (await readFile(spec.ref, 'utf8')).replace(/\r?\n$/, ''); + if (typeof resolveSecret !== 'function') return undefined; + return resolveSecret(spec.name, spec); +}; + +/** + * The grant.issued entry for grantMeta. Its scope is the room grant this run + * acts under (exact tool names, a connection id), as the host knows it; the + * run grant's own credential names and patterns show up in credential.* and + * in each permission.decided rule. `taskId` + `podId` bind the grant to that + * board task, ending when it is done unless `endsOn` says otherwise. + */ +const grantIssued = (meta) => { + const { + grantId = randomUUID(), parentGrantId, scope = {}, target, issuedBy, expiresAt, taskId, podId, endsOn = 'task.done', + } = meta; + return { + grantId, + ...(parentGrantId === undefined ? {} : { parentGrantId }), + scope: { tools: [], audience: [], ...scope }, + ...(target ? { target } : {}), + ...(issuedBy ? { issuedBy } : {}), + ...(expiresAt ? { expiresAt: expiresAt instanceof Date ? expiresAt.toISOString() : expiresAt } : {}), + ...(taskId ? { binding: { taskId, podId, endsOn } } : {}), + }; +}; + +/** + * prepareRunEnv({ grant, baseEnv, resolveSecret, root?, require?, trace?, ctx?, grantMeta? }) + * → { env, home, cleanup, granted: [names], grantId? } + * + * `require` lists credential names the adapter cannot run without; one the + * grant does not name fails the run as credential_denied before any spawn. + * A named credential that cannot be resolved, or that resolves to fewer than + * MIN_CREDENTIAL_LENGTH characters, fails it the same way: a run that + * silently lacks its key fails later and less legibly, and a value that short + * would slip past the refusal-text reader's redaction. + * + * Events go to `trace`, else ctx.trace: grant.issued (only with grantMeta, + * first), then credential.granted {names} or credential.denied {name, reason}. + * Values never enter an event or a message, and a hook that offers + * redact(values) is handed the resolved values so the record can scrub them + * out of agent text too. + */ +export const prepareRunEnv = async ({ + grant, baseEnv = {}, resolveSecret, root, require: required = [], trace, ctx, grantMeta, +} = {}) => { + const valid = validateGrant(grant); + const hook = trace && typeof trace.emit === 'function' ? trace : traceOf(ctx); + const issued = grantMeta ? grantIssued(grantMeta) : null; + if (issued) hook.emit('grant.issued', issued); + const realHomes = [...new Set([baseEnv.HOME, homedir()].filter(Boolean))]; + const DENIED = { + not_granted: 'is not in the run grant', + unresolved: 'could not be resolved', + too_short: `resolved to a value shorter than ${MIN_CREDENTIAL_LENGTH} characters`, + }; + const deny = (name, reason, values) => { + hook.emit('credential.denied', { name, reason }); + return runFailure('credential_denied', `${name} ${DENIED[reason]}`, { credentials: values }); + }; + + const granted = new Set(valid.credentials.map((c) => c.name)); + const missing = required.find((name) => !granted.has(name)); + if (missing) throw deny(missing, 'not_granted', []); + + const { home, dirs } = await makeHome(root); + const cleanup = () => rm(home, { recursive: true, force: true }); + const env = { ...pickBaseEnv(baseEnv, realHomes), HOME: home, ...dirs }; + const values = []; + try { + for (const spec of valid.credentials) { + let value; + // A resolver's error text may quote the secret it failed on; drop it. + try { value = await resolveOne(spec, { baseEnv, resolveSecret }); } catch { value = undefined; } + if (!usable(value)) throw deny(spec.name, 'unresolved', values); + // A value this short cannot be told apart from ordinary text, so the + // refusal-text reader would not redact it. Refuse it here, before spawn. + if (value.length < MIN_CREDENTIAL_LENGTH) throw deny(spec.name, 'too_short', values); + values.push(value); + env[spec.name] = value; + } + } catch (error) { + await cleanup(); + throw error; + } + if (typeof hook.redact === 'function') hook.redact(values); + const names = valid.credentials.map((c) => c.name); + hook.emit('credential.granted', { names }); + return { env, home, cleanup, granted: names, ...(issued ? { grantId: issued.grantId } : {}) }; +}; diff --git a/packages/seats/src/valet/grant.js b/packages/seats/src/valet/grant.js new file mode 100644 index 0000000000..b2109bfd39 --- /dev/null +++ b/packages/seats/src/valet/grant.js @@ -0,0 +1,118 @@ +/** + * A grant says what one run may hold and do: which credentials reach its env, + * and which tool calls it may make without asking, only after asking, or never. + * + * { + * credentials: [{ name: 'OPENAI_API_KEY', from: 'env'|'file'|'resolver', ref? }], + * tools: { allow: [pattern], ask: [pattern], deny: [pattern] }, + * } + * + * `from: 'env'` copies baseEnv[ref ?? name]; `from: 'file'` reads the absolute + * path in `ref`; `from: 'resolver'` asks the host's resolveSecret(name, spec), + * with `ref` as an opaque hint for it. + * + * A tool pattern is `` or `:`, where kind is one of + * ACP's ToolKind values or `*`. `execute` matches every execute call; + * `execute:git status*` matches execute calls whose title starts that way; + * `*:*.env*` matches any kind by title. Globs know `*` and `?` only. + * + * The title is text the agent chose. A title-glob deny narrows what the agent + * can do; a title-glob allow is only as strong as the agent's honesty about + * its title, so prefer kind-level allows and put execute behind ask. + * + * Precedence is deny > ask > allow, and a call no rule names is denied. + * + * `network` rules are refused for now rather than accepted and ignored: a run + * would believe it was fenced when nothing enforces the fence. TODO: egress + * allowlist once a sandbox can enforce it. + */ + +export const TOOL_KINDS = Object.freeze(['read', 'edit', 'delete', 'move', 'search', 'execute', 'think', 'fetch', 'other']); +export const CREDENTIAL_SOURCES = Object.freeze(['env', 'file', 'resolver']); +const RULE_LISTS = ['deny', 'ask', 'allow']; // checked in this order: the precedence +const VALIDATED = Symbol('validated grant'); + +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/; +// Names the run env is built from. A credential under one of these names would +// overwrite the boundary itself (a granted HOME points back at the real login). +const RESERVED_NAME = /^(?:HOME|PATH|TMPDIR|USER|LOGNAME|SHELL|PWD|LANG|LC_[A-Z0-9_]+|TERM|TZ|NO_COLOR|CI|SSL_CERT_FILE|SSL_CERT_DIR|XDG_[A-Z0-9_]+)$/; + +const fail = (message) => { throw new Error(`grant: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); + } +}; + +const globToRegExp = (glob) => new RegExp(`^${glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[\\s\\S]*').replace(/\?/g, '[\\s\\S]')}$`); + +/** Parse once at validation so matching never re-reads a malformed pattern. */ +const parsePattern = (pattern, where) => { + if (typeof pattern !== 'string' || !pattern.trim()) fail(`${where} must be a non-empty string`); + const colon = pattern.indexOf(':'); + const kind = colon === -1 ? pattern : pattern.slice(0, colon); + const title = colon === -1 ? null : pattern.slice(colon + 1); + if (kind !== '*' && !TOOL_KINDS.includes(kind)) fail(`${where} names unknown tool kind "${kind}"`); + if (title !== null && !title) fail(`${where} has an empty title glob`); + return Object.freeze({ pattern, kind, title: title === null ? null : globToRegExp(title) }); +}; + +const validateCredential = (credential, index, seen) => { + const where = `credentials[${index}]`; + if (!isPlainObject(credential)) fail(`${where} must be an object`); + refuseUnknownKeys(credential, ['name', 'from', 'ref'], `${where}.`); + const { name, from, ref } = credential; + if (typeof name !== 'string' || !ENV_NAME.test(name)) fail(`${where}.name must be an ENV_NAME (A-Z, 0-9, _)`); + if (RESERVED_NAME.test(name)) fail(`${where}.name ${name} is reserved for the run env`); + if (seen.has(name)) fail(`${where}.name ${name} is granted twice`); + seen.add(name); + if (!CREDENTIAL_SOURCES.includes(from)) fail(`${where}.from must be one of ${CREDENTIAL_SOURCES.join(', ')}`); + if (ref !== undefined && (typeof ref !== 'string' || !ref)) fail(`${where}.ref must be a non-empty string`); + if (from === 'env' && ref !== undefined && !ENV_NAME.test(ref)) fail(`${where}.ref must be an ENV_NAME for from: env`); + if (from === 'file' && (ref === undefined || !ref.startsWith('/'))) fail(`${where}.ref must be an absolute path for from: file`); + return Object.freeze(ref === undefined ? { name, from } : { name, from, ref }); +}; + +/** Validate and freeze a grant. Throws on the first problem; never repairs one. */ +export const validateGrant = (grant) => { + if (grant && grant[VALIDATED]) return grant; + if (!isPlainObject(grant)) fail('must be an object'); + if ('network' in grant) fail('network rules are not enforced yet; remove the field rather than rely on it'); + refuseUnknownKeys(grant, ['credentials', 'tools'], ''); + const credentials = grant.credentials ?? []; + if (!Array.isArray(credentials)) fail('credentials must be an array'); + const seen = new Set(); + const tools = grant.tools ?? {}; + if (!isPlainObject(tools)) fail('tools must be an object'); + refuseUnknownKeys(tools, RULE_LISTS, 'tools.'); + const rules = {}; + for (const list of RULE_LISTS) { + const patterns = tools[list] ?? []; + if (!Array.isArray(patterns)) fail(`tools.${list} must be an array`); + rules[list] = Object.freeze(patterns.map((pattern, i) => parsePattern(pattern, `tools.${list}[${i}]`))); + } + return Object.freeze({ + [VALIDATED]: true, + credentials: Object.freeze(credentials.map((c, i) => validateCredential(c, i, seen))), + tools: Object.freeze(rules), + }); +}; +const matches = (parsed, kind, title) => (parsed.kind === '*' || parsed.kind === kind) + && (parsed.title === null || parsed.title.test(title)); + +/** + * Which rule decides this tool call. ACP makes `kind` optional and means + * `other` when it is absent, so that is what an unknown or missing kind is. + * Returns { decision: 'allow'|'ask'|'deny', rule: { list, pattern } }. + */ +export const matchToolCall = (grant, toolCall = {}) => { + const valid = validateGrant(grant); + const kind = TOOL_KINDS.includes(toolCall?.kind) ? toolCall.kind : 'other'; + const title = typeof toolCall?.title === 'string' ? toolCall.title : ''; + for (const list of RULE_LISTS) { + const hit = valid.tools[list].find((parsed) => matches(parsed, kind, title)); + if (hit) return { decision: list, rule: { list, pattern: hit.pattern } }; + } + return { decision: 'deny', rule: { list: 'default', pattern: null } }; +}; diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js new file mode 100644 index 0000000000..a6445688c1 --- /dev/null +++ b/packages/seats/src/valet/permissions.js @@ -0,0 +1,213 @@ +/** + * Turn a grant into ctx.decidePermission: the answer to ACP's + * session/request_permission, decided per tool call. + * + * allow → the allow_once option + * deny → the reject_once option, or cancelled when there is none + * ask → approve({ toolCall, grantRule }) from the host. Only an answer + * that says who gave it decides: + * { approved: true | false, by: <user id>, role, decisionCardId? }, + * with `role` an owner role. Anything else (a bare true, 'yes', + * {approved: 1}, an answer naming nobody, a throw, no answer in + * time) is a failed ask, and the policy's default, deny, decides. + * An answer whose role is `pod-member` decides nothing (ADR-033 D5): + * the call stays parked, and approve is called again with + * `refused: { by, role: 'pod-member', reason: 'not_owner' }` so the + * card can show why and wait for the owner. + * + * allow_always and reject_always are never chosen: either would be remembered + * by the agent past this grant, so the next call would skip the check. + * An allow rule with no allow_once option therefore denies. + */ +import { traceOf } from '../trace/events.js'; +import { argsDigestOf, PARKED_CALL_ROLES, isUserId, policyActorId } from '../record/format.js'; +import { matchToolCall, validateGrant } from './grant.js'; + +export const DEFAULT_APPROVE_TIMEOUT_MS = 5 * 60 * 1000; +// An approver that keeps handing back pod-member answers is broken; after this +// many the ask fails, and the policy's default decides, as for any broken approver. +export const MAX_REFUSED_ANSWERS = 10; + +const optionOfKind = (options, kind) => (Array.isArray(options) ? options : []) + .find((option) => option && option.kind === kind && typeof option.optionId === 'string'); + +const allowAnswer = (options) => { + const option = optionOfKind(options, 'allow_once'); + return option ? { outcome: 'selected', optionId: option.optionId, kind: option.kind } : null; +}; +const denyAnswer = (options) => { + const option = optionOfKind(options, 'reject_once'); + return option ? { outcome: 'selected', optionId: option.optionId, kind: option.kind } : { outcome: 'cancelled' }; +}; + +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isText = (value) => typeof value === 'string' && value.length > 0; + +// The roles a person can answer a parked call as; `policy` is not a person. +const ANSWER_ROLES = PARKED_CALL_ROLES.filter((role) => role !== 'policy'); + +/** + * What an approver's answer says. It approves or rejects only when it is an + * object whose `approved` is exactly true or false and that names a user id + * and an owner role. Any other answer approves nothing: it is a failed ask, + * so the record never carries a decision nobody can be held to. + */ +const readAnswer = (answer) => { + if (!isPlainObject(answer) || typeof answer.approved !== 'boolean') return { verdict: 'failed' }; + if (!isUserId(answer.by) || !ANSWER_ROLES.includes(answer.role)) return { verdict: 'failed' }; + return { + verdict: answer.approved ? 'approved' : 'rejected', + by: answer.by, + role: answer.role, + ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), + }; +}; + +const isPodMemberAnswer = (answer) => isPlainObject(answer) && answer.role === 'pod-member'; + +/** + * Wait for the host's approval, bounded. Resolves { verdict, by?, role?, + * decisionCardId? }, verdict 'approved' | 'rejected' | 'timed_out' | 'failed' + * | 'aborted'; never rejects, so a broken approver becomes a denial rather + * than a hung or crashed turn. + */ +const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((done) => { + if (typeof approve !== 'function') { done({ verdict: 'failed' }); return; } + if (signal?.aborted) { done({ verdict: 'aborted' }); return; } + let finished = false; + const finish = (verdict) => { + if (finished) return; + finished = true; + clearTimeout(timer); + signal?.removeEventListener?.('abort', onAbort); + done(verdict); + }; + const onAbort = () => finish({ verdict: 'aborted' }); + const timer = setTimeout(() => finish({ verdict: 'timed_out' }), timeoutMs); + signal?.addEventListener?.('abort', onAbort, { once: true }); + let refusals = 0; + const ask = (current) => Promise.resolve() + .then(() => approve(current)) + .then((answer) => { + if (finished) return; + if (!isPodMemberAnswer(answer)) { finish(readAnswer(answer)); return; } + // A pod member may not decide a parked call: the call stays parked and + // the owner is asked again, with the refusal, inside the same timeout. + refusals += 1; + if (refusals > MAX_REFUSED_ANSWERS) { finish({ verdict: 'failed' }); return; } + const refused = { by: isUserId(answer.by) ? answer.by : null, role: 'pod-member', reason: 'not_owner' }; + setTimeout(() => { if (!finished) ask({ ...input, refused }); }, 0); + }, () => finish({ verdict: 'failed' })); + ask(input); +}); + +/** + * Who decided an ask, for the record. A person who answered is named by the + * approver; anything else (no answer in time, an answer naming nobody, a + * broken approver, a cancelled run) means the policy's default decided. + * Rule-decided calls carry no decidedBy here; the record writer fills in + * "policy" for any decision whose rule list is not `ask`. + */ +const decider = (approval) => { + if (approval.verdict === 'approved' || approval.verdict === 'rejected') { + return { + decidedBy: approval.by, + decidedByRole: approval.role, + ...(approval.decisionCardId ? { decisionCardId: approval.decisionCardId } : {}), + }; + } + return { decidedBy: 'policy', decidedByRole: 'policy' }; +}; + +const isAsker = (value) => isPlainObject(value) && isUserId(value.id) && PARKED_CALL_ROLES.includes(value.role); + +/** + * Who an ask goes to, for permission.requested. `askedOf` is {id, role} or + * ({toolCall, grantRule}) => {id, role, decisionCardId?}. A call a + * rule decides is asked of the grant's policy. A function that throws or + * answers malformed is recorded as asked of the policy too: the record does + * not name a person it cannot vouch for. + */ +const askOf = (askedOf, input, policy) => { + if (input.grantRule?.list !== 'ask') return policy; + let answer = askedOf; + if (typeof askedOf === 'function') { + try { answer = askedOf(input); } catch { answer = null; } + } + if (!isAsker(answer)) return policy; + return { + id: answer.id, + role: answer.role, + ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), + }; +}; + +/** + * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal?, grantId?, askedOf? }) + * → (request) => Promise<{outcome:'selected', optionId} | {outcome:'cancelled'}> + * + * `signal` is the run's AbortSignal: a cancelled run answers a pending ask + * with cancelled, which is what ACP asks of a client after session/cancel. + * `grantId` is recorded on each decision, to tie it to its grant.issued. + * With `askedOf` (above), each request is recorded first as + * permission.requested: the question (the tool call's title), who it is + * asked of, and the card it rides on. The approver receives that card id. + * When the tool call carries `rawInput`, both entries carry its argsDigest. + * Given a `trace`, the returned function has `recordsPermissions === true`. + */ +export const createPermissionPolicy = (grant, { + approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, grantId, askedOf, +} = {}) => { + const valid = validateGrant(grant); + const hook = traceOf({ trace }); + if (askedOf !== undefined && typeof askedOf !== 'function' && !isAsker(askedOf)) { + throw new Error('askedOf must be {id: <user id>, role: <decidedByRole>} or a function returning one'); + } + const policy = { id: policyActorId(grantId), role: 'policy' }; + const decide = async (request = {}) => { + const toolCall = request?.toolCall || {}; + const options = request?.options; + const { decision, rule } = matchToolCall(valid, toolCall); + const toolCallId = typeof toolCall.toolCallId === 'string' ? toolCall.toolCallId : null; + const argsDigest = argsDigestOf(toolCall.rawInput); + let asker = null; + if (askedOf !== undefined) { + asker = askOf(askedOf, { toolCall, grantRule: rule }, policy); + const fallback = `${isText(toolCall.kind) ? toolCall.kind : 'tool'} call`; + hook.emit('permission.requested', { + toolCallId, + options: (Array.isArray(options) ? options : []).filter(isPlainObject).map(({ optionId, kind }) => ({ optionId, kind })), + question: isText(toolCall.title) ? toolCall.title : fallback, + askedOf: { id: asker.id, role: asker.role }, + ...(grantId ? { grantId } : {}), + ...(asker.decisionCardId ? { decisionCardId: asker.decisionCardId } : {}), + ...(argsDigest ? { argsDigest } : {}), + }); + } + let approval; + let answer; + if (decision === 'ask') { + const input = { toolCall, grantRule: rule, ...(asker?.decisionCardId ? { decisionCardId: asker.decisionCardId } : {}) }; + approval = await awaitApproval(approve, input, { timeoutMs: approveTimeoutMs, signal }); + if (approval.verdict === 'aborted') answer = { outcome: 'cancelled' }; + else answer = approval.verdict === 'approved' ? allowAnswer(options) : null; + } else if (decision === 'allow') { + answer = allowAnswer(options); + } + if (!answer) answer = denyAnswer(options); + hook.emit('permission.decided', { + outcome: answer.outcome, + ...(answer.optionId ? { optionId: answer.optionId, kind: answer.kind } : {}), + toolCallId, + rule, + ...(approval ? { approval: approval.verdict, ...decider(approval) } : {}), + ...(grantId ? { grantId } : {}), + ...(argsDigest ? { argsDigest } : {}), + }); + return answer.outcome === 'selected' ? { outcome: 'selected', optionId: answer.optionId } : { outcome: 'cancelled' }; + }; + // With a trace this writes both permission entries itself, so an adapter + // handed it as ctx.decidePermission must not write them again. + if (trace) Object.defineProperty(decide, 'recordsPermissions', { value: true }); + return decide; +}; diff --git a/packages/seats/test-support/fake-provider.js b/packages/seats/test-support/fake-provider.js new file mode 100644 index 0000000000..62bb98bf77 --- /dev/null +++ b/packages/seats/test-support/fake-provider.js @@ -0,0 +1,83 @@ +/** + * A local OpenAI-compatible model server for tests and the compatibility + * matrix. Real CLIs (goose, opencode, hermes, …) point their provider base URL + * here and run a genuine turn with no key and no network. + * + * Behaviour is chosen per request by a directive in the last user message: + * [[reply:TEXT]] answer TEXT (default: PONG) + * [[stall]] accept the request and never answer (liveness tests) + * [[refuse]] answer HTTP 400 with a content-policy error body + * [[fail:CODE]] answer HTTP CODE + * Serves POST /v1/chat/completions (stream and non-stream) and GET /v1/models. + */ +import { createServer } from 'node:http'; + +const directiveOf = (body) => { + const messages = Array.isArray(body?.messages) ? body.messages : []; + const text = messages.filter((m) => m?.role === 'user').map((m) => ( + typeof m.content === 'string' ? m.content + : Array.isArray(m.content) ? m.content.map((p) => p?.text || '').join('') : '' + )).pop() || ''; + const match = /\[\[(reply|stall|refuse|fail)(?::([^\]]*))?\]\]/.exec(text); + return match ? { kind: match[1], arg: match[2] ?? '' } : { kind: 'reply', arg: '' }; +}; + +export const startFakeProvider = async ({ model = 'fake-model', defaultReply = 'PONG', port = 0 } = {}) => { + const requests = []; + const open = new Set(); + const server = createServer((req, res) => { + open.add(res); + res.on('close', () => open.delete(res)); + let raw = ''; + req.on('data', (chunk) => { raw += chunk; }); + req.on('end', () => { + if (req.method === 'GET' && /\/models$/.test(req.url)) { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ object: 'list', data: [{ id: model, object: 'model', owned_by: 'fake' }] })); + return; + } + if (req.method !== 'POST' || !/\/chat\/completions$/.test(req.url)) { + res.writeHead(404); res.end(); return; + } + let body = {}; + try { body = JSON.parse(raw || '{}'); } catch { /* answer as default */ } + const directive = directiveOf(body); + requests.push({ url: req.url, model: body.model, stream: !!body.stream, directive: directive.kind, authorization: req.headers.authorization ? 'present' : 'absent' }); + if (directive.kind === 'stall') return; // hold the socket open, never answer + if (directive.kind === 'refuse') { + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'The request was rejected by the content policy.', type: 'invalid_request_error', code: 'content_policy_violation' } })); + return; + } + if (directive.kind === 'fail') { + res.writeHead(Number(directive.arg) || 500, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'fake provider failure', type: 'server_error' } })); + return; + } + const text = directive.arg || defaultReply; + const id = `chatcmpl-fake-${requests.length}`; + const created = Math.floor(Date.now() / 1000); + const usage = { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }; + if (body.stream) { + res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }); + const chunk = (delta, finish = null, extra = {}) => res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`); + chunk({ role: 'assistant', content: '' }); + chunk({ content: text }); + chunk({}, 'stop', body.stream_options?.include_usage ? {} : { usage }); + if (body.stream_options?.include_usage) res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [], usage })}\n\n`); + res.end('data: [DONE]\n\n'); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ id, object: 'chat.completion', created, model: body.model || model, choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }], usage })); + }); + }); + await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); + const { port: bound } = server.address(); + return { + baseURL: `http://127.0.0.1:${bound}/v1`, + model, + requests, + close: () => new Promise((resolve) => { for (const res of open) res.destroy(); server.close(() => resolve()); }), + }; +}; diff --git a/packages/seats/test-support/processes.js b/packages/seats/test-support/processes.js new file mode 100644 index 0000000000..5e2811f8c4 --- /dev/null +++ b/packages/seats/test-support/processes.js @@ -0,0 +1,30 @@ +// Real-process helpers for the kill tests. +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; + +// A zombie still answers kill(pid, 0); on Linux it is read as gone, because a +// SIGKILLed orphan waits there until init reaps it. +export const isRunning = (pid) => { + try { process.kill(pid, 0); } catch (error) { return error.code !== 'ESRCH'; } + try { return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return true; } +}; + +export const waitFor = async (predicate, ms = 5000) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (await predicate()) return true; + await new Promise((r) => { setTimeout(r, 25); }); + } + return predicate(); +}; + +/** The comma-separated pids a process wrote to `file`, once it has written them. */ +export const readPids = async (file, count = 1) => { + const ready = () => existsSync(file) && new RegExp(`^\\d+(,\\d+){${count - 1}}\\s*$`).test(readFileSync(file, 'utf8')); + assert.ok(await waitFor(ready), `no ${count} pid(s) in ${file}`); + return readFileSync(file, 'utf8').trim().split(',').map(Number); +}; + +export const readPid = async (file) => (await readPids(file, 1))[0]; + +export const reap = (pid) => { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }; diff --git a/packages/seats/test-support/record-sample.js b/packages/seats/test-support/record-sample.js new file mode 100644 index 0000000000..ba3bc3102b --- /dev/null +++ b/packages/seats/test-support/record-sample.js @@ -0,0 +1,156 @@ +/** + * Builds docs/examples: one task, two people, two agents on different CLIs. + * + * Ana's opencode agent fixes TASK-101 under a grant that attenuates Ben's + * Sentry connection grant. Its first attempt hangs, and Ana rules on a card + * to retry it; a `git push` is denied by the grant; a Sentry write is asked + * of Ben on a decision card, and he approves it as the credential's owner. Ben's goose agent + * then reviews the PR, and the board marks the task done, which ends the + * review grant bound to it. + * + * Everything runs through the real store and valet, with a fixed clock and + * fake credential values, so the files are reproducible byte for byte. + */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { toOtlpJson } from '../src/record/otel.js'; +import { argsDigestOf } from '../src/record/format.js'; + +export const SAMPLE_SECRETS = Object.freeze(['sk-or-sample-0000aaaa1111bbbb', 'sntry-proxy-sample-2222cccc3333', 'sk-sample-openai-4444dddd5555']); +export const SAMPLE_RUN_IDS = Object.freeze(['run-ana-0001', 'run-ben-0001']); +const POD = 'pod-checkout'; +const TASK = 'TASK-101'; +const OPTIONS = [ + { optionId: 'allow', name: 'Allow', kind: 'allow_once' }, + { optionId: 'always', name: 'Always allow', kind: 'allow_always' }, + { optionId: 'reject', name: 'Reject', kind: 'reject_once' }, +]; + +/** + * A tool call the way an ACP adapter would record it (with the digest of its + * arguments, never the arguments), then decided by the run's policy, which + * records the request and the decision. + */ +const toolCall = async (run, decide, { rawInput, ...call }, finalStatus) => { + run.emit('tool.call', { ...call, status: 'pending', argsDigest: argsDigestOf(rawInput) }); + const answer = await decide({ + sessionId: 'ses', toolCall: { toolCallId: call.toolCallId, title: call.title, kind: call.kind, rawInput }, options: OPTIONS, + }); + const allowed = answer.outcome === 'selected' && answer.optionId === 'allow'; + run.emit('tool.update', { toolCallId: call.toolCallId, status: allowed ? finalStatus : 'failed' }); +}; + +export const buildSample = async (root) => { + let tick = Date.UTC(2026, 9, 25, 16, 0, 0); + const store = createTraceStore({ root: join(root, 'trace'), now: () => new Date((tick += 1000)) }); + const [secretOr, secretSentry, secretOpenai] = SAMPLE_SECRETS; + + // Ana's agent: opencode, retried once, one denied call, one approved by Ben. + const ana = store.startRun({ + runId: SAMPLE_RUN_IDS[0], adapter: 'opencode', taskId: TASK, podId: POD, + actor: { type: 'agent', id: 'u-agent-ana-opencode', owner: 'u-ana', name: "Ana's opencode" }, + }); + ana.emit('task.claimed', { leaseUntil: '2026-10-25T18:00:00Z' }); + const anaGrant = { + credentials: [{ name: 'OPENROUTER_API_KEY', from: 'env' }, { name: 'SENTRY_PROXY_TOKEN', from: 'resolver' }], + tools: { allow: ['read', 'search', 'other:commonly_*'], ask: ['fetch'], deny: ['execute:git push*'] }, + }; + const anaEnv = await prepareRunEnv({ + grant: anaGrant, + baseEnv: { OPENROUTER_API_KEY: secretOr }, + resolveSecret: async () => secretSentry, + root, + ctx: { trace: ana }, + grantMeta: { + grantId: 'grant-ana-sentry-1', + parentGrantId: 'grant-ben-sentry-root', + scope: { + connectionId: 'conn-ben-sentry', tools: ['sentry.get_issue', 'sentry.resolve_issue'], writeMode: 'write-with-confirm', + budget: { calls: 20, windowMs: 3600000 }, audience: ['u-ana', 'u-ben'], + }, + target: { kind: 'seat', id: 'u-agent-ana-opencode' }, + issuedBy: 'u-ana', + expiresAt: '2026-10-25T20:00:00.000Z', + taskId: TASK, + podId: POD, + }, + }); + await anaEnv.cleanup(); + const anaDecide = createPermissionPolicy(anaGrant, { + trace: ana, + grantId: 'grant-ana-sentry-1', + // An ask under this grant goes to Ben, who owns the Sentry connection, on a card. + askedOf: () => ({ id: 'u-ben', role: 'credential-owner', decisionCardId: 'card-7f3' }), + approve: async ({ decisionCardId }) => ({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId }), + }); + + ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, attempt: 1 }); + ana.emit('turn.prompt_sent', { chars: 412 }); + ana.emit('tool.call', { toolCallId: 'tc-1', title: 'Read src/checkout/total.js', kind: 'read', status: 'pending', surface: 'native' }); + ana.emit('tool.update', { toolCallId: 'tc-1', status: 'completed' }); + ana.emit('liveness.stalled', { idleMs: 90000, windowMs: 90000 }); + ana.emit('liveness.recovery', { action: 'cancel', attempt: 1 }); + ana.emit('attempt.settled', { attempt: 1, outcome: 'stuck', failure: { kind: 'stuck', detail: 'no progress signal for 90000ms (window 90000ms)' } }); + ana.emit('decision.ruled', { decisionCardId: 'card-2b9', value: 'retry', decidedBy: 'u-ana', decidedByRole: 'agent-owner' }); + ana.emit('liveness.recovery', { action: 'retry', attempt: 2 }); + + ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, sessionId: 'ses-ana-2', attempt: 2 }); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-2', title: 'git push origin fix/total-rounding', kind: 'execute', surface: 'native', + rawInput: { command: 'git push origin fix/total-rounding' }, + }, 'completed'); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-3', title: 'Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212', kind: 'fetch', + name: 'sentry.resolve_issue', surface: 'connector', rawInput: { issueId: 'CHECKOUT-88', status: 'resolved' }, + }, 'completed'); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-4', title: 'commonly_post_message: the fix summary, to the pod', kind: 'other', name: 'commonly_post_message', surface: 'commonly', + rawInput: { podId: 'pod-checkout', content: 'Fixed in #212' }, + }, 'completed'); + ana.emit('agent.message_chunk', { text: 'Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open.' }); + ana.emit('usage', { inputTokens: 18234, outputTokens: 2210 }); + ana.emit('attempt.settled', { attempt: 2, outcome: 'ok' }); + ana.emit('result.reported', { status: 'done', refs: [{ type: 'pr', id: '#212' }] }); + ana.emit('run.settled', { outcome: 'ok', durationMs: 412000, attempts: 2 }); + + // Ben's agent: goose, read-only review; the board then marks the task done. + const ben = store.startRun({ + runId: SAMPLE_RUN_IDS[1], adapter: 'goose', taskId: TASK, podId: POD, + actor: { type: 'agent', id: 'u-agent-ben-goose', owner: 'u-ben', name: "Ben's goose" }, + }); + const benGrant = { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }], tools: { allow: ['read', 'search'] } }; + const benEnv = await prepareRunEnv({ + grant: benGrant, + baseEnv: { OPENAI_API_KEY: secretOpenai }, + root, + ctx: { trace: ben }, + grantMeta: { + grantId: 'grant-ben-review-1', + scope: { connectionId: 'conn-ben-github', tools: ['github.get_pull_request'], writeMode: 'read', audience: ['u-ben'] }, + target: { kind: 'seat', id: 'u-agent-ben-goose' }, + issuedBy: 'u-ben', + expiresAt: '2026-10-25T20:00:00.000Z', + taskId: TASK, + podId: POD, + }, + }); + await benEnv.cleanup(); + const benDecide = createPermissionPolicy(benGrant, { trace: ben, grantId: 'grant-ben-review-1', askedOf: { id: 'u-ben', role: 'agent-owner' } }); + ben.emit('run.started', { adapter: 'goose', cli: { name: 'goose', version: '1.54.0' }, attempt: 1 }); + await toolCall(ben, benDecide, { + toolCallId: 'tc-1', title: 'Read pull request #212', kind: 'read', name: 'github.get_pull_request', surface: 'connector', + rawInput: { pullNumber: 212 }, + }, 'completed'); + ben.emit('agent.message_chunk', { text: 'Reviewed #212: the rounding fix is right and the half-cent case is tested.' }); + ben.emit('attempt.settled', { attempt: 1, outcome: 'ok' }); + ben.emit('task.done', { doneBy: 'u-ben' }); + ben.emit('run.settled', { outcome: 'ok', durationMs: 95000, attempts: 1 }); + + const files = Object.fromEntries(SAMPLE_RUN_IDS.map((runId) => [`${runId}.jsonl`, readFileSync(join(root, 'trace', 'runs', `${runId}.jsonl`), 'utf8')])); + const entries = SAMPLE_RUN_IDS.flatMap((runId) => store.readRun(runId)); + files[`${TASK}.otlp.json`] = `${JSON.stringify(toOtlpJson(entries, { serviceName: 'seats-sample' }), null, 2)}\n`; + return { store, files, entries }; +}; diff --git a/packages/seats/test/args-digest-vectors.test.js b/packages/seats/test/args-digest-vectors.test.js new file mode 100644 index 0000000000..2d0b2c3a91 --- /dev/null +++ b/packages/seats/test/args-digest-vectors.test.js @@ -0,0 +1,60 @@ +/** + * test/vectors/args-digest.json is the contract a writer in another language + * passes to produce the same argsDigest as argsDigestOf. This test holds the + * vectors to the code, so neither can change without the other. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { argsDigestOf } from '../src/record/format.js'; + +const FILE = join(dirname(fileURLToPath(import.meta.url)), 'vectors', 'args-digest.json'); +const { _about: about, vectors } = JSON.parse(readFileSync(FILE, 'utf8')); + +/** The file's two tags: {"$undefined":true} and {"$number":"<literal>"}, each an object with one key. */ +const decode = (value) => { + if (Array.isArray(value)) return value.map(decode); + if (value === null || typeof value !== 'object') return value; + const keys = Object.keys(value); + if (keys.length === 1 && keys[0] === '$undefined' && value.$undefined === true) return undefined; + if (keys.length === 1 && keys[0] === '$number' && typeof value.$number === 'string') return Number(value.$number); + return Object.fromEntries(keys.map((key) => [key, decode(value[key])])); +}; + +test('the vector file documents its tags and covers what writers get wrong', () => { + assert.match(about, /\$undefined/); + assert.match(about, /\$number/); + const names = new Set(vectors.map((v) => v.name)); + assert.equal(names.size, vectors.length, 'vector names are distinct'); + for (const v of vectors) { + assert.equal(typeof v.name, 'string'); + assert.ok('args' in v, `${v.name} has args`); + if (v.error === true) { + assert.deepEqual(Object.keys(v).sort(), ['args', 'error', 'name'], `${v.name}: an error vector has no digest`); + } else { + assert.match(v.digest, /^sha256:[0-9a-f]{64}$/, v.name); + assert.equal(typeof v.canonical, 'string', v.name); + } + } + const raw = JSON.stringify(vectors); + for (const tag of ['{"$undefined":true}', '{"$number":"NaN"}', '{"$number":"-0"}', '{"$number":"1e21"}', '{"$number":"1.0"}']) { + assert.ok(raw.includes(tag), `a vector uses ${tag}`); + } + assert.ok(vectors.filter((v) => v.error).length >= 3, 'non-finite and undefined inputs are covered'); + assert.ok(vectors.some((v) => v.error && /lone surrogate/.test(v.name)), 'a lone surrogate is a no-digest case'); +}); + +for (const v of vectors) { + test(`args digest vector: ${v.name}`, () => { + const got = argsDigestOf(decode(v.args)); + if (v.error === true) { + assert.equal(got, undefined, 'no digest: a writer omits argsDigest'); + return; + } + assert.equal(v.digest, `sha256:${createHash('sha256').update(v.canonical, 'utf8').digest('hex')}`, 'digest is sha256 of canonical'); + assert.equal(got, v.digest); + }); +} diff --git a/packages/seats/test/record-example.test.js b/packages/seats/test/record-example.test.js new file mode 100644 index 0000000000..931b25c91f --- /dev/null +++ b/packages/seats/test/record-example.test.js @@ -0,0 +1,84 @@ +/** + * docs/examples is generated from test-support/record-sample.js. This test + * fails when the files drift from what the code writes; regenerate them with + * SEATS_WRITE_EXAMPLES=1 node --test test/record-example.test.js + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildSample, SAMPLE_RUN_IDS, SAMPLE_SECRETS } from '../test-support/record-sample.js'; +import { verifyRecord } from '../src/record/format.js'; + +const EXAMPLES = join(dirname(fileURLToPath(import.meta.url)), '..', 'docs', 'examples'); + +const built = async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-example-')); + try { + const { files, store } = await buildSample(root); + return { files, verified: SAMPLE_RUN_IDS.map((runId) => store.verifyRun(runId)) }; + } finally { rmSync(root, { recursive: true, force: true }); } +}; + +test('docs/examples matches what the code writes', async () => { + const { files, verified } = await built(); + for (const result of verified) assert.deepEqual(result.errors, []); + if (process.env.SEATS_WRITE_EXAMPLES === '1') { + mkdirSync(EXAMPLES, { recursive: true }); + for (const [name, body] of Object.entries(files)) writeFileSync(join(EXAMPLES, name), body); + } + for (const [name, body] of Object.entries(files)) { + const path = join(EXAMPLES, name); + assert.ok(existsSync(path), `${name} is missing; run with SEATS_WRITE_EXAMPLES=1`); + assert.equal(readFileSync(path, 'utf8'), body, `${name} drifted; run with SEATS_WRITE_EXAMPLES=1`); + } +}); + +test('the committed examples verify on their own and hold no secret value', () => { + for (const runId of SAMPLE_RUN_IDS) { + const text = readFileSync(join(EXAMPLES, `${runId}.jsonl`), 'utf8'); + const result = verifyRecord(text); + assert.deepEqual(result.errors, []); + assert.equal(result.settled, true); + } + const all = SAMPLE_RUN_IDS.map((id) => `${id}.jsonl`).concat('TASK-101.otlp.json') + .map((name) => readFileSync(join(EXAMPLES, name), 'utf8')).join('\n'); + for (const secret of SAMPLE_SECRETS) assert.ok(!all.includes(secret)); +}); + +test('the headline case: Ana\'s agent asks, Ben approves as the credential owner', () => { + const entries = readFileSync(join(EXAMPLES, 'run-ana-0001.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + const asked = entries.find((e) => e.kind === 'permission.decided' && e.data.approval === 'approved'); + const agent = entries.find((e) => e.kind === 'attempt.started').actor; + assert.equal(agent.owner, 'u-ana'); + assert.equal(asked.data.decidedBy, 'u-ben'); + assert.notEqual(asked.data.decidedBy, agent.owner, 'the approver is not the agent\'s owner'); + assert.equal(asked.data.decidedByRole, 'credential-owner'); + assert.equal(asked.data.decisionCardId, 'card-7f3'); + assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }); + const issued = entries.find((e) => e.kind === 'grant.issued'); + assert.equal(asked.data.grantId, issued.data.grantId); + // The ask itself: Ana's agent asks Ben, tied to the grant and the card, about this exact call. + const request = entries.find((e) => e.kind === 'permission.requested' && e.data.toolCallId === asked.data.toolCallId); + assert.equal(request.actor.owner, 'u-ana'); + assert.deepEqual(request.data.askedOf, { id: 'u-ben', role: 'credential-owner' }); + assert.equal(request.data.grantId, issued.data.grantId); + assert.equal(request.data.decisionCardId, 'card-7f3'); + assert.match(request.data.question, /CHECKOUT-88/); + const call = entries.find((e) => e.kind === 'tool.call' && e.data.toolCallId === asked.data.toolCallId); + assert.match(call.data.argsDigest, /^sha256:[0-9a-f]{64}$/); + assert.equal(request.data.argsDigest, call.data.argsDigest); + assert.equal(asked.data.argsDigest, call.data.argsDigest); + const ruled = entries.find((e) => e.kind === 'decision.ruled'); + assert.deepEqual(ruled.data, { decidedBy: 'u-ana', decidedByRole: 'agent-owner', decisionCardId: 'card-2b9', value: 'retry' }); + assert.equal(issued.data.scope.connectionId, 'conn-ben-sentry'); + assert.equal(issued.data.parentGrantId, 'grant-ben-sentry-root'); + const denied = entries.find((e) => e.kind === 'permission.decided' && e.data.rule.list === 'deny'); + assert.deepEqual([denied.data.decidedBy, denied.data.decidedByRole], ['policy', 'policy']); + const ben = readFileSync(join(EXAMPLES, 'run-ben-0001.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + const revoked = ben.find((e) => e.kind === 'grant.revoked'); + assert.deepEqual(revoked.data, { grantIds: ['grant-ben-review-1'], reason: 'task.done' }); + assert.equal(ben[ben.indexOf(revoked) - 1].kind, 'task.done'); +}); diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js new file mode 100644 index 0000000000..beebd91b24 --- /dev/null +++ b/packages/seats/test/record-format.test.js @@ -0,0 +1,559 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, SANDBOXES, SESSION_METHODS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + RETRY_ERROR_TYPES, RETRY_STATES, +} from '../src/record/format.js'; + +const RUN = 'run-fmt-0001'; +const ENV = { runId: RUN, taskId: 'TASK-7', podId: 'pod-a' }; +const AGENT = { type: 'agent', id: 'u-agent-1', owner: 'u-ana', name: "Ana's agent" }; +let clock = Date.UTC(2026, 9, 25, 12, 0, 0); +const ts = () => new Date((clock += 1000)).toISOString(); + +/** A small, settled record built through the writer: [lines], each without '\n'. */ +const build = (count = 6) => { + const lines = []; + const add = (kind, data, extra = {}) => { + lines.push(appendEntry(lines.at(-1) ?? null, { ...ENV, ts: ts(), kind, data, ...extra }, { runActor: AGENT })); + }; + add('task.claimed', {}); + add('attempt.started', { attempt: 1, adapter: 'fake' }); + for (let i = 0; i < count - 3; i += 1) add('tool.call', { toolCallId: `tc-${i}`, title: `step ${i}`, surface: 'native' }); + add('run.settled', { outcome: 'ok', durationMs: 10, attempts: 1 }); + return lines; +}; +const text = (lines) => lines.map((line) => `${line}\n`).join(''); +const reasons = (result) => result.errors.map(({ seq, reason }) => `${seq}:${reason}`); + +test('canonical form: sorted keys, no whitespace, absent fields omitted', () => { + assert.equal(canonicalLine({ b: 1, a: { d: [1, 'x'], c: null }, e: undefined }), '{"a":{"c":null,"d":[1,"x"]},"b":1}'); + // UTF-16 code-unit order, as RFC 8785 asks: 'B' (0x42) sorts before 'a' (0x61). + assert.equal(canonicalLine({ a: 1, B: 2 }), '{"B":2,"a":1}'); + assert.throws(() => canonicalLine({ x: Number.NaN }), /finite/); +}); + +test('round trip: what the writer wrote verifies, reads back, and chains by the exact bytes', () => { + const lines = build(); + const result = verifyRecord(text(lines)); + assert.deepEqual(result.errors, []); + assert.equal(result.ok, true); + assert.equal(result.entries, lines.length); + assert.equal(result.settled, true); + assert.equal(result.head, createHash('sha256').update(lines.at(-1)).digest('hex')); + const entries = readRecord(text(lines)); + assert.equal(entries.length, lines.length); + entries.forEach((entry, i) => { + assert.equal(entry.v, RECORD_VERSION); + assert.equal(entry.schema, RECORD_SCHEMA); + assert.equal(entry.seq, i + 1); + assert.equal(entry.prev, i === 0 ? null : hashLine(lines[i - 1])); + assert.equal(canonicalLine(entry), lines[i], 'each line is the canonical form of its entry'); + assert.equal(entry.taskId, 'TASK-7'); + assert.equal(entry.podId, 'pod-a'); + }); + assert.deepEqual(entries[0].actor, { type: 'agent', id: 'u-agent-1', owner: 'u-ana', name: "Ana's agent" }); + assert.deepEqual(entries.at(-1).actor, { type: 'system', id: 'seats' }, 'run.settled defaults to the system actor'); +}); + +test('tamper: an edited byte is reported at the next seq as prev_mismatch', () => { + const lines = build(); + // A byte inside seq 3's data: still JSON, still canonical, so only the chain sees it. + lines[2] = lines[2].replace('step 0', 'step 9'); + const result = verifyRecord(text(lines)); + assert.equal(result.ok, false); + assert.deepEqual(reasons(result), ['4:prev_mismatch']); +}); + +test('tamper: an edited byte that breaks canonical form is reported on its own line too', () => { + const lines = build(); + lines[2] = lines[2].replace('"kind":"tool.call"', '"kind": "tool.call"'); + assert.deepEqual(reasons(verifyRecord(text(lines))), ['3:not_canonical', '4:prev_mismatch']); +}); + +test('tamper: a deleted line is a seq gap and a prev mismatch at the line after it', () => { + const lines = build(); + lines.splice(2, 1); // seq 3 gone + const result = verifyRecord(text(lines)); + assert.deepEqual(reasons(result), ['4:seq_mismatch', '4:prev_mismatch']); +}); + +test('tamper: deleting the first line is caught at seq 2', () => { + const lines = build(); + lines.shift(); + assert.deepEqual(reasons(verifyRecord(text(lines))), ['2:seq_mismatch', '2:prev_mismatch']); +}); + +test('tamper: swapped lines are reported where the order breaks', () => { + const lines = build(); + [lines[2], lines[3]] = [lines[3], lines[2]]; // seq 4 now sits before seq 3 + const result = verifyRecord(text(lines)); + assert.equal(result.ok, false); + assert.deepEqual(reasons(result), ['4:seq_mismatch', '4:prev_mismatch', '3:seq_mismatch', '3:prev_mismatch', '5:seq_mismatch', '5:prev_mismatch']); +}); + +test('tamper: a torn final line is reported at the seq it would have had', () => { + const lines = build(); + const torn = `${text(lines.slice(0, -1))}${lines.at(-1).slice(0, 25)}`; + const result = verifyRecord(torn); + assert.deepEqual(reasons(result), [`${lines.length}:torn`]); + assert.equal(result.entries, lines.length - 1); + assert.equal(result.settled, false); + assert.equal(readRecord(torn).length, lines.length - 1, 'readers skip the torn tail'); +}); + +test('a record cut short at a line boundary is an unsettled prefix, not tampering', () => { + const result = verifyRecord(text(build().slice(0, 3))); + assert.equal(result.ok, true); + assert.equal(result.settled, false); +}); + +test('a line after run.settled, and a line from another run, are reported', () => { + const lines = build(); + const extra = JSON.parse(lines.at(-1)); + const after = canonicalLine({ ...extra, seq: extra.seq + 1, kind: 'usage', data: {}, prev: hashLine(lines.at(-1)) }); + assert.deepEqual(reasons(verifyRecord(text([...lines, after]))), [`${extra.seq + 1}:after_settled`]); + assert.throws(() => appendEntry(lines.at(-1), { ...ENV, ts: ts(), kind: 'usage', data: {} }), /after run.settled/); + const mid = JSON.parse(lines[1]); + lines[1] = canonicalLine({ ...mid, runId: 'run-other-01' }); + assert.ok(reasons(verifyRecord(text(lines))).includes('2:run_mismatch')); +}); + +test('a newer v is refused, and unknown kinds still chain', () => { + const first = JSON.parse(build()[0]); + const v2 = canonicalLine({ ...first, v: 2 }); + assert.deepEqual(reasons(verifyRecord(`${v2}\n`)), ['1:unsupported_version']); + const unknown = canonicalLine({ ...first, kind: 'future.thing' }); + const next = canonicalLine({ ...JSON.parse(build()[1]), prev: hashLine(unknown) }); + assert.deepEqual(verifyRecord(`${unknown}\n${next}\n`).errors, []); +}); + +test('podId is required whenever taskId is set: writer and verifier', () => { + assert.throws(() => appendEntry(null, { runId: RUN, taskId: 'TASK-7', ts: ts(), kind: 'task.claimed', data: {} }), /taskId needs a podId/); + // A podId alone is fine; so is neither. + appendEntry(null, { runId: RUN, podId: 'pod-a', ts: ts(), kind: 'task.claimed', data: {} }); + appendEntry(null, { runId: RUN, ts: ts(), kind: 'attempt.started', data: { attempt: 1 } }); + const line = canonicalLine({ + v: 1, schema: RECORD_SCHEMA, runId: RUN, taskId: 'TASK-7', seq: 1, ts: ts(), kind: 'task.claimed', actor: null, data: {}, prev: null, + }); + assert.deepEqual(verifyRecord(`${line}\n`).errors, [{ seq: 1, reason: 'bad_envelope', detail: 'podId' }]); +}); + +test('taskId and podId are fixed for the run', () => { + const first = appendEntry(null, { ...ENV, ts: ts(), kind: 'task.claimed', data: {} }); + assert.throws(() => appendEntry(first, { ...ENV, podId: 'pod-b', ts: ts(), kind: 'usage', data: {} }), /fixed for the whole run/); +}); + +test('actor.id is an id, never a display name; owner is a user id on agents only', () => { + const at = (actor) => appendEntry(null, { runId: RUN, ts: ts(), kind: 'usage', data: {}, actor }); + assert.throws(() => at({ type: 'agent', id: "Ana's agent" }), /actor.id/); + assert.throws(() => at({ type: 'human', id: 'Ben Smith' }), /actor.id/); + assert.throws(() => at({ type: 'human', id: 'u-ben', owner: 'u-ana' }), /owner/); + assert.throws(() => at({ type: 'robot', id: 'u-x' }), /actor.type/); + const entry = JSON.parse(at({ type: 'human', id: 'u-ben', name: 'Ben Smith' })); + assert.deepEqual(entry.actor, { id: 'u-ben', name: 'Ben Smith', type: 'human' }); + assert.equal(JSON.parse(at(null)).actor, null, 'null records "unknown" and is kept'); +}); + +const grantData = (overrides = {}) => ({ + grantId: 'g-1', + scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], writeMode: 'read', budget: { calls: 5, windowMs: 60000 }, audience: ['u-ana'] }, + issuedBy: 'u-ana', + ...overrides, +}); +const issue = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'grant.issued', data }); + +test('grant.issued: scope matches a room grant, with exact tool names and a connection id', () => { + const entry = JSON.parse(issue(grantData({ + parentGrantId: 'g-root', target: { kind: 'seat', id: 'u-agent-1' }, expiresAt: '2026-10-26T00:00:00Z', + binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.done' }, + }))); + assert.deepEqual(entry.data.scope, grantData().scope); + assert.equal(entry.data.parentGrantId, 'g-root'); + assert.deepEqual(entry.actor, { type: 'human', id: 'u-ana' }, 'the issuer is the default actor'); + // Patterns, a credential name in place of a connection, and unknown fields are refused. + for (const tools of [['sentry.*'], ['execute:git push*'], ['read?'], ['two words']]) { + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, tools } })), /exact tool names/, `tools ${tools}`); + } + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, credentials: ['SENTRY_TOKEN'] } })), /unknown field grant.issued.scope.credentials/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, writeMode: 'admin' } })), /writeMode/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, budget: { calls: -1 } } })), /budget.calls/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, audience: ['Ana Smith'] } })), /audience/); + assert.throws(() => issue(grantData({ secret: 'x' })), /unknown field grant.issued.secret/); + assert.equal(JSON.parse(issue(grantData({ issuedBy: 'policy' }))).actor.id, 'grant:g-1'); +}); + +test('grant.issued binding: names the run\'s own task and pod, and ends on task.done or expiresAt', () => { + issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.done' } })); + issue(grantData({ expiresAt: '2026-10-26T00:00:00Z', binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'expiresAt' } })); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.closed' } })), /endsOn/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'expiresAt' } })), /needs an expiresAt/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', endsOn: 'task.done' } })), /taskId and podId/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-8', podId: 'pod-a', endsOn: 'task.done' } })), /run's own/); +}); + +test('grant.revoked: a non-empty list of grant ids and a reason from the enum', () => { + const revoke = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'grant.revoked', data }); + for (const reason of ['task.done', 'expired', 'cascade', 'manual']) revoke({ grantIds: ['g-1', 'g-2'], reason }); + assert.throws(() => revoke({ grantIds: [], reason: 'manual' }), /non-empty/); + assert.throws(() => revoke({ grantId: 'g-1', reason: 'manual' }), /grantIds/); + assert.throws(() => revoke({ grantIds: ['g-1', 'g-1'], reason: 'manual' }), /repeats/); + for (const reason of ['task.closed', 'run.settled', 'revoked_by:u-ana', undefined]) { + assert.throws(() => revoke({ grantIds: ['g-1'], reason }), /reason/, `reason ${reason}`); + } +}); + +test('task.done: written by the board, and only in a run with a task', () => { + const entry = JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'task.done', data: { doneBy: 'u-ben' } })); + assert.deepEqual(entry.actor, { type: 'system', id: 'board' }); + assert.throws(() => appendEntry(null, { runId: RUN, ts: ts(), kind: 'task.done', data: {} }), /needs a run with a taskId/); + assert.equal(KINDS['task.done'].actor, 'board'); +}); + +test('permission.decided: decidedBy is separate from the agent owner, and credential-owner is a role', () => { + const decide = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data }, { runActor: AGENT })); + const asked = decide({ + outcome: 'selected', optionId: 'allow', kind: 'allow_once', toolCallId: 'tc-1', rule: { list: 'ask', pattern: 'fetch' }, + approval: 'approved', decidedBy: 'u-ben', decidedByRole: 'credential-owner', decisionCardId: 'card-1', + }); + assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }, 'the decider is the actor, not the agent or its owner'); + assert.equal(asked.data.decidedByRole, 'credential-owner'); + for (const role of ['decision-owner', 'agent-owner']) decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: role }); + decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'policy' }); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'policy' }), /exactly when/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'credential-owner' }), /exactly when/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'owner' }), /decidedByRole/); + // A parked call is decided by an owner or by policy, never a pod member (ADR-033 D5). + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-cal', decidedByRole: 'pod-member' }), /decidedByRole/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'Ben Smith' }), /decidedBy/); + const byRule = decide({ outcome: 'selected', optionId: 'reject', kind: 'reject_once', toolCallId: 't', rule: { list: 'deny', pattern: 'x' }, grantId: 'g-1' }); + assert.equal(byRule.data.decidedBy, 'policy'); + assert.equal(byRule.data.decidedByRole, 'policy'); + assert.deepEqual(byRule.actor, { type: 'policy', id: 'grant:g-1' }); + // A decision always names who made it; no writer may record one nobody can be held to. + const asked2 = { outcome: 'selected', optionId: 'allow', toolCallId: 't', rule: { list: 'ask', pattern: 'x' }, approval: 'approved' }; + assert.throws(() => decide(asked2), /decidedBy is required/); + assert.throws(() => decide({ ...asked2, decidedBy: null, decidedByRole: null }), /decidedBy is required/); + assert.throws(() => decide({ ...asked2, decidedBy: 'u-ben', decidedByRole: null }), /decidedByRole/); + assert.throws(() => decide({ ...asked2, decidedBy: 'u-ben' }), /decidedByRole/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't' }), /decidedBy is required/, 'no rule, no decider'); +}); + +test('tool.call surface is one of connector, commonly, native', () => { + const call = (surface) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', surface } }); + for (const surface of ['connector', 'commonly', 'native']) assert.equal(JSON.parse(call(surface)).data.surface, surface); + assert.throws(() => call('mcp'), /surface/); + assert.equal(toolSurfaceOf('commonly_post_message'), 'commonly'); + assert.equal(toolSurfaceOf('mcp__commonly__commonly_read'), 'commonly'); + assert.equal(toolSurfaceOf('mcp__sentry__resolve_issue'), 'connector'); + assert.equal(toolSurfaceOf('Bash'), 'native'); +}); + +test('credential entries hold names only', () => { + const granted = (names) => appendEntry(null, { ...ENV, ts: ts(), kind: 'credential.granted', data: { names } }); + granted(['OPENAI_API_KEY']); + assert.throws(() => granted(['sk-live-0123456789']), /NAMES/); + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'credential.denied', data: { name: 'sk-abc', reason: 'unresolved' } }), /NAMES/); +}); + +test('the writer refuses unknown kinds, a bad seq, and a non-UTC ts', () => { + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'made.up', data: {} }), /unknown record kind/); + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'usage', data: {}, seq: 2 }), /seq 2/); + assert.throws(() => appendEntry(null, { ...ENV, ts: '2026-10-25T12:00:00+02:00', kind: 'usage', data: {} }), /RFC 3339/); +}); + +const request = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.requested', data }, { runActor: AGENT })); +const askData = (overrides = {}) => ({ + toolCallId: 'tc-1', options: [{ optionId: 'allow', kind: 'allow_once' }], question: 'Resolve CHECKOUT-88?', + askedOf: { id: 'u-ben', role: 'credential-owner' }, grantId: 'g-1', decisionCardId: 'card-1', ...overrides, +}); + +test('permission.requested: question and askedOf are required, askedOf.id is an id and its role is from the enum', () => { + const entry = request(askData()); + assert.deepEqual(entry.data.askedOf, { id: 'u-ben', role: 'credential-owner' }); + assert.equal(entry.data.question, 'Resolve CHECKOUT-88?'); + assert.deepEqual(entry.actor, AGENT, 'the agent asks'); + const { askedOf, question, ...rest } = askData(); + assert.throws(() => request(rest), /question/); + assert.throws(() => request({ ...rest, question }), /askedOf/); + assert.throws(() => request(askData({ question: '' })), /question/); + assert.throws(() => request(askData({ askedOf: { id: 'Ben Smith', role: 'credential-owner' } })), /askedOf.id/); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben' } })), /askedOf.role/); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'boss' } })), /askedOf.role/); + assert.throws(() => request(askData({ askedOf: { id: 'u-cal', role: 'pod-member' } })), /askedOf.role/, 'a parked call is not asked of a pod member'); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'policy', email: 'ben@example.com' } })), /unknown field/); + assert.throws(() => request(askData({ decisionCardId: '' })), /decisionCardId/); + assert.equal(request(askData({ askedOf: { id: 'grant:g-1', role: 'policy' } })).data.askedOf.id, 'grant:g-1'); +}); + +test('decision.ruled: a card ruling, validated like permission.decided', () => { + const rule = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'decision.ruled', data })); + const good = { decisionCardId: 'card-2', value: 'retry', decidedBy: 'u-ana', decidedByRole: 'agent-owner' }; + const entry = rule(good); + assert.deepEqual(entry.data, good); + assert.deepEqual(entry.actor, { type: 'human', id: 'u-ana' }, 'the ruler is the actor'); + assert.deepEqual(rule({ ...good, decidedBy: 'policy', decidedByRole: 'policy' }).actor, { type: 'policy', id: 'policy' }); + for (const value of [true, 3, 'ship']) rule({ ...good, value }); + const byMember = rule({ ...good, decidedBy: 'u-cal', decidedByRole: 'pod-member' }); + assert.equal(byMember.data.decidedByRole, 'pod-member', 'a pod member may rule a card that parks no call'); + assert.deepEqual(byMember.actor, { type: 'human', id: 'u-cal' }); + assert.throws(() => rule({ ...good, decisionCardId: undefined }), /decisionCardId/); + assert.throws(() => rule({ ...good, decidedBy: 'Ana Smith' }), /decidedBy/); + assert.throws(() => rule({ ...good, decidedBy: undefined }), /decidedBy/); + assert.throws(() => rule({ ...good, decidedByRole: 'owner' }), /decidedByRole/); + assert.throws(() => rule({ ...good, decidedByRole: null }), /decidedByRole/); + for (const value of [undefined, null, { a: 1 }, ['x'], Number.NaN]) assert.throws(() => rule({ ...good, value }), /value/, `value ${value}`); +}); + +test('argsDigest is sha256 of the canonical args, and anything else is refused, so raw args cannot ride in', () => { + const digest = argsDigestOf({ b: 2, a: [1, 'x'] }); + assert.equal(digest, `sha256:${createHash('sha256').update('{"a":[1,"x"],"b":2}').digest('hex')}`); + assert.equal(argsDigestOf({ a: [1, 'x'], b: 2 }), digest, 'key order does not change it'); + assert.equal(argsDigestOf(undefined), undefined); + assert.equal(argsDigestOf({ n: Number.NaN }), undefined); + assert.equal(argsDigestOf({ s: '\ud800' }), undefined, 'a lone surrogate has no digest (RFC 8785)'); + assert.equal(argsDigestOf({ ['k\udc00']: 1 }), undefined, 'in a key too'); + const call = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', argsDigest } }); + const decided = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data: { outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'policy', argsDigest } }); + for (const write of [call, decided, (d) => request(askData({ argsDigest: d }))]) { + write(digest); + for (const raw of [{ command: 'git push' }, 'git push origin main', `sha256:${'A'.repeat(64)}`, `sha256:${'a'.repeat(63)}`, `sha512:${'a'.repeat(64)}`, 'a'.repeat(64), ['x'], 42]) { + assert.throws(() => write(raw), /argsDigest/, `raw ${JSON.stringify(raw)}`); + } + } +}); + +const BEHALF = { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester' } }; + +test('actor.onBehalfOf: an agent entry carrying it hashes, chains and verifies, on every kind the agent authors', () => { + assert.deepEqual(ON_BEHALF_ROLES, ['requester', 'assigner']); + const lines = []; + const add = (kind, data) => lines.push(appendEntry(lines.at(-1) ?? null, { ...ENV, ts: ts(), kind, data }, { runActor: BEHALF })); + add('attempt.started', { attempt: 1, adapter: 'fake' }); + add('tool.call', { toolCallId: 'tc-1', title: 'resolve', surface: 'connector', argsDigest: argsDigestOf({ id: 88 }) }); + add('permission.requested', askData({ argsDigest: argsDigestOf({ id: 88 }) })); + lines.push(appendEntry(lines.at(-1), { + ...ENV, ts: ts(), kind: 'result.reported', data: { status: 'done' }, actor: { ...AGENT, onBehalfOf: { id: 'u-cy', role: 'assigner' } }, + })); + const result = verifyRecord(text(lines)); + assert.deepEqual(result.errors, []); + assert.equal(result.head, hashLine(lines.at(-1))); + const entries = readRecord(text(lines)); + for (const entry of entries.slice(0, 3)) assert.deepEqual(entry.actor, BEHALF, `${entry.kind} carries the pair on its actor`); + assert.deepEqual(entries[3].actor.onBehalfOf, { id: 'u-cy', role: 'assigner' }); + assert.equal(entries[2].data.onBehalfOf, undefined, 'the pair lives on the actor, never in data'); + assert.match(lines[1], /"actor":\{"id":"u-agent-1","name":"Ana's agent","onBehalfOf":\{"id":"u-ben","role":"requester"\},"owner":"u-ana","type":"agent"\}/); +}); + +test('actor.onBehalfOf: refused off agent actors, with a bad id, an unknown role or an extra key, by writer and verifier', () => { + const at = (actor) => appendEntry(null, { runId: RUN, ts: ts(), kind: 'usage', data: {}, actor }); + const bad = [ + [{ type: 'human', id: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } }, /agent actors only/], + [{ type: 'system', id: 'seats', onBehalfOf: { id: 'u-ben', role: 'requester' } }, /agent actors only/], + [{ ...AGENT, onBehalfOf: { id: 'Ben Smith', role: 'requester' } }, /onBehalfOf.id/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben' } }, /onBehalfOf.role/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben', role: 'owner' } }, /onBehalfOf.role/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester', email: 'ben@example.com' } }, /unknown field actor.onBehalfOf.email/], + [{ ...AGENT, onBehalfOf: 'u-ben' }, /onBehalfOf must be/], + [{ ...AGENT, onBehalfOf: null }, /onBehalfOf must be/], + ]; + const good = JSON.parse(at(BEHALF)); + for (const [actor, message] of bad) { + assert.throws(() => at(actor), message, JSON.stringify(actor)); + // A hand-written line with the same actor, canonical and chained, fails the envelope check. + const line = canonicalLine({ ...good, actor }); + assert.deepEqual(reasons(verifyRecord(`${line}\n`)), ['1:bad_envelope'], JSON.stringify(actor)); + assert.equal(verifyRecord(`${line}\n`).errors[0].detail, 'actor'); + } +}); + +test('actor.onBehalfOf is additive: a record written without it is byte-identical to the writer before the field', () => { + // Hashes pinned from the writer at b7b71f63a, before onBehalfOf existed. + const env = { runId: 'run-fmt-0001', taskId: 'TASK-7', podId: 'pod-a' }; + const l1 = appendEntry(null, { ...env, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', data: { attempt: 1, adapter: 'fake' } }, { runActor: AGENT }); + const l2 = appendEntry(l1, { ...env, ts: '2026-10-25T12:00:02.000Z', kind: 'tool.call', data: { toolCallId: 'tc-1', title: 'x', surface: 'native' } }, { runActor: AGENT }); + assert.equal(hashLine(l1), 'f91b90fc46781ca9b915b80cce5b9633f603e90742bfc8b1f6dc82d17e01c264'); + assert.equal(hashLine(l2), '1689f865735018eafddcb3fcf0b267d8ead7d9546d45b6d9a471c23eed21cfc6'); + assert.ok(!l1.includes('onBehalfOf') && !l2.includes('onBehalfOf')); + assert.deepEqual(verifyRecord(`${l1}\n${l2}\n`).errors, []); +}); + +test('a lone surrogate in agent text still records: only the args digest refuses it', () => { + // A stream cut mid-pair leaves half a surrogate pair in a chunk. + const chunk = `cut here ${'\ud83d'}`; + const first = appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.message_chunk', data: { text: chunk } }, { runActor: AGENT }); + assert.ok(first.includes('cut here \\ud83d'), 'the lone half is written as a \\u escape'); + const second = appendEntry(first, { + ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 'tc-1', title: chunk, argsDigest: argsDigestOf({ q: 'ok' }) }, + }, { runActor: AGENT }); + assert.deepEqual(verifyRecord(`${first}\n${second}\n`).errors, []); + assert.equal(readRecord(`${first}\n`)[0].data.text, chunk); + assert.equal(argsDigestOf({ q: chunk }), undefined); +}); + +const egress = (kind, data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind, data }, { runActor: AGENT })); + +test('egress.allowed: {host, port}, written by the system', () => { + assert.deepEqual(KINDS['egress.allowed'], { group: 'egress', actor: 'system' }); + const ok = egress('egress.allowed', { host: 'api.sentry.io', port: 443 }); + assert.deepEqual(ok.data, { host: 'api.sentry.io', port: 443 }); + assert.deepEqual(ok.actor, { type: 'system', id: 'seats' }); + // host: a lowercase DNS name, an IP literal, or [redacted]; anything else is refused. + for (const host of ['localhost', '[redacted]', '203.0.113.7', '2001:db8::1']) egress('egress.allowed', { host, port: 443 }); + for (const host of ['API.sentry.io', 'https://api.sentry.io', 'user@api.sentry.io', 'a b.io', '-x.io', '', '[REDACTED]', 42]) { + assert.throws(() => egress('egress.allowed', { host, port: 443 }), /egress.allowed.host/, `host ${JSON.stringify(host)}`); + } + assert.throws(() => egress('egress.allowed', { host: `${'a'.repeat(60)}.`.repeat(5), port: 443 }), /egress.allowed.host/, 'over 253 characters'); + // port: an integer from 1 to 65535. + egress('egress.allowed', { host: 'api.sentry.io', port: 65535 }); + for (const port of [0, 65536, 443.5, '443', -1]) assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io', port }), /egress.allowed.port/, `port ${port}`); + // An opened tunnel always names its target: no nulls, no missing fields. + assert.throws(() => egress('egress.allowed', { host: null, port: null }), /needs a host and a port/); + assert.throws(() => egress('egress.allowed', { port: 443 }), /egress.allowed.host/); + assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io' }), /egress.allowed.port/); + // No other field: a header or an address can't ride in. + assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io', port: 443, address: '203.0.113.7' }), /unknown field egress.allowed.address/); +}); + +test('egress.denied: {host, port, reason}, the reason from a closed set, null target only on a malformed request', () => { + assert.deepEqual(KINDS['egress.denied'], { group: 'egress', actor: 'system' }); + assert.deepEqual(EGRESS_DENIED_REASONS, ['bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable']); + const denied = egress('egress.denied', { host: 'evil.example', port: 443, reason: 'not_allowed' }); + assert.deepEqual(denied.data, { host: 'evil.example', port: 443, reason: 'not_allowed' }); + assert.deepEqual(denied.actor, { type: 'system', id: 'seats' }); + // reason: every listed reason is accepted; anything else is refused. + for (const reason of EGRESS_DENIED_REASONS) egress('egress.denied', { host: '10.0.0.1', port: 80, reason }); + for (const reason of ['forbidden', undefined, null, 'NOT_ALLOWED']) { + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 443, reason }), /egress.denied.reason/, `reason ${reason}`); + } + // host and port may be null, together, when the request was malformed. + for (const reason of ['bad_request', 'not_connect']) { + assert.deepEqual(egress('egress.denied', { host: null, port: null, reason }).data, { host: null, port: null, reason }); + } + assert.throws(() => egress('egress.denied', { host: null, port: null, reason: 'not_allowed' }), /needs a host and a port/, 'a parsed target is never null'); + assert.throws(() => egress('egress.denied', { host: null, port: 443, reason: 'bad_request' }), /null together/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: null, reason: 'not_connect' }), /null together/); + // host and port follow the egress.allowed rules when set. + egress('egress.denied', { host: '[redacted]', port: 443, reason: 'not_allowed' }); + assert.throws(() => egress('egress.denied', { host: 'Evil.example', port: 443, reason: 'not_allowed' }), /egress.denied.host/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 70000, reason: 'not_allowed' }), /egress.denied.port/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 443, reason: 'not_allowed', status: 403 }), /unknown field egress.denied.status/); +}); + +test('sandbox.applied: {sandbox, reason?}, the reason only when the run is unsandboxed', () => { + assert.deepEqual(KINDS['sandbox.applied'], { group: 'sandbox', actor: 'system' }); + assert.deepEqual(SANDBOXES, ['seatbelt', 'bwrap', 'none']); + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'sandbox.applied', data }, { runActor: AGENT })); + const applied = at({ sandbox: 'seatbelt' }); + assert.deepEqual(applied.data, { sandbox: 'seatbelt' }); + assert.deepEqual(applied.actor, { type: 'system', id: 'seats' }); + // sandbox: one of seatbelt, bwrap, none. + at({ sandbox: 'bwrap' }); + for (const sandbox of ['docker', undefined, null, 'Seatbelt']) assert.throws(() => at({ sandbox }), /sandbox.applied.sandbox/, `sandbox ${sandbox}`); + // reason: a non-empty string, and only beside none. + assert.deepEqual(at({ sandbox: 'none', reason: 'no supported OS sandbox on win32' }).data, { sandbox: 'none', reason: 'no supported OS sandbox on win32' }); + at({ sandbox: 'none' }); + assert.throws(() => at({ sandbox: 'seatbelt', reason: 'because' }), /only when sandbox is none/); + for (const reason of ['', 42, null]) assert.throws(() => at({ sandbox: 'none', reason }), /sandbox.applied.reason/, `reason ${reason}`); + // No other field. + assert.throws(() => at({ sandbox: 'bwrap', egress: 'proxy' }), /unknown field sandbox.applied.egress/); +}); + +test('session.opened: {method, sessionId, requestedSessionId?, fallback?}, the pair only when a new session replaced a requested one', () => { + assert.deepEqual(KINDS['session.opened'], { group: 'attempt', actor: 'system' }); + assert.deepEqual(SESSION_METHODS, ['new', 'resume', 'load']); + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'session.opened', data }, { runActor: AGENT })); + const opened = at({ method: 'resume', sessionId: 'ses-1' }); + assert.deepEqual(opened.data, { method: 'resume', sessionId: 'ses-1' }); + assert.deepEqual(opened.actor, { type: 'system', id: 'seats' }); + // method: new, resume or load. + at({ method: 'load', sessionId: 'ses-1' }); + for (const method of ['open', undefined, 'NEW']) assert.throws(() => at({ method, sessionId: 'ses-1' }), /session.opened.method/, `method ${method}`); + // sessionId: required, a non-empty string. + at({ method: 'new', sessionId: '01JB5Z' }); + for (const sessionId of [undefined, '', 7]) assert.throws(() => at({ method: 'new', sessionId }), /needs a sessionId/, `sessionId ${sessionId}`); + // requestedSessionId and fallback: together, and only on a new session. + const replaced = { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1' }; + assert.deepEqual(at({ ...replaced, fallback: 'session/resume answered error -32603' }).data, { ...replaced, fallback: 'session/resume answered error -32603' }); + at({ ...replaced, fallback: 'agent supports neither session/resume nor session/load' }); + at({ ...replaced, fallback: 'session/load answered error 404' }); + assert.throws(() => at(replaced), /come together/); + assert.throws(() => at({ method: 'new', sessionId: 'ses-2', fallback: 'session/load answered error 404' }), /come together/); + assert.throws(() => at({ ...replaced, method: 'resume', fallback: 'session/load answered error 404' }), /only when method is new/); + assert.throws(() => at({ ...replaced, requestedSessionId: '', fallback: 'session/load answered error 404' }), /requestedSessionId must be/); + // fallback: one of the two forms the adapter writes, nothing else. + for (const fallback of ['it broke', 'session/new answered error 1', 'session/load answered error x', 'session/load answered error 1; rm -rf', null, '']) { + assert.throws(() => at({ ...replaced, fallback }), /session.opened.fallback/, `fallback ${JSON.stringify(fallback)}`); + } + assert.throws(() => at({ method: 'new', sessionId: 'ses-2', cwd: '/home/ana' }), /unknown field session.opened.cwd/); +}); + +test('usage: contextTokens and contextSize are integers >= 0, or null when unknown', () => { + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'usage', data }, { runActor: AGENT })); + const filled = at({ inputTokens: null, outputTokens: null, totalTokens: null, cost: null, contextTokens: 41000, contextSize: 200000 }); + assert.equal(filled.data.contextTokens, 41000); + assert.equal(filled.data.contextSize, 200000); + at({ contextTokens: 0, contextSize: null }); + at({ inputTokens: 10, outputTokens: 2 }); + for (const key of ['contextTokens', 'contextSize']) { + for (const value of [-1, 1.5, '41000', Number.NaN]) assert.throws(() => at({ [key]: value }), new RegExp(`usage.${key}`), `${key} ${value}`); + } +}); + +test('agent.retry: state, attempt, nextRetryAt, status and errorType, each absent when unknown', () => { + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT })); + const full = at({ state: 'scheduled', attempt: 2, nextRetryAt: '2026-10-25T13:00:02.000Z', status: 503, errorType: 'provider.internal' }); + assert.equal(full.kind, 'agent.retry'); + assert.equal(full.actor.id, AGENT.id); + at({ state: 'started', attempt: 2 }); + at({ state: 'started' }); + for (const errorType of RETRY_ERROR_TYPES) at({ state: 'scheduled', errorType }); + assert.ok(RETRY_ERROR_TYPES.includes('other')); + for (const state of [undefined, 'waiting', 'SCHEDULED']) assert.throws(() => at({ state }), /agent.retry.state/, String(state)); + for (const attempt of [0, -1, 1.5, '2', null]) assert.throws(() => at({ state: 'started', attempt }), /agent.retry.attempt/, String(attempt)); + for (const nextRetryAt of ['soon', '2026-10-25 13:00:02', 12, null]) assert.throws(() => at({ state: 'scheduled', nextRetryAt }), /agent.retry.nextRetryAt/, String(nextRetryAt)); + // Provider text has no field to ride in. + for (const key of ['message', 'body', 'response', 'error']) { + assert.throws(() => at({ state: 'scheduled', [key]: 'upstream said no' }), new RegExp(`unknown field agent.retry.${key}`)); + } + assert.deepEqual(RETRY_STATES, ['scheduled', 'started']); +}); + +test('agent.retry.errorType is a closed enum: any other value is refused', () => { + // opencode 2.1.4's provider error types, read from the installed package, + // plus `other`. Changing this set is a record-v1 change, so it is pinned here. + assert.deepEqual([...RETRY_ERROR_TYPES], [ + 'provider.auth', 'provider.content-filter', 'provider.free-limit', 'provider.internal', + 'provider.invalid-output', 'provider.invalid-request', 'provider.no-route', 'provider.quota', + 'provider.rate-limit', 'provider.timeout', 'provider.transport', 'provider.unknown', + 'provider.unsupported-operation', 'other', + ]); + assert.ok(Object.isFrozen(RETRY_ERROR_TYPES)); + const at = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT }); + for (const errorType of ['provider.made-up', 'Provider.Internal', 'provider.internal ', 'server_error', 'upstream said sk-live-1234', '', null, 42, ['provider.internal']]) { + assert.throws(() => at({ state: 'scheduled', errorType }), /agent.retry.errorType must be one of/, JSON.stringify(errorType)); + } +}); + +test('agent.retry.status is an integer HTTP status from 100 to 599, or absent', () => { + const at = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT }); + at({ state: 'scheduled', status: 100 }); + at({ state: 'scheduled', status: 599 }); + at({ state: 'scheduled' }); + for (const status of [99, 600, 0, -503, 503.5, '503', null, Number.NaN]) { + assert.throws(() => at({ state: 'scheduled', status }), /agent.retry.status/, String(status)); + } +}); + +test('record-v1.md states the hook attribution rule and lists every agent.retry error type', () => { + const doc = readFileSync(fileURLToPath(new URL('../docs/record-v1.md', import.meta.url)), 'utf8').replace(/\s+/g, ' '); + assert.match(doc, /A permission hook's answer is held to the same rule\. An answer that does not name who decided \(`decidedBy` a user id, plus an owner `decidedByRole`\) gets the reject default, recorded as `decidedBy`\/`decidedByRole` `policy`\/`policy`/); + assert.match(doc, /A decider marked `recordsPermissions` \(the valet's policy\) writes and attributes its own entries, so it is exempt\./); + const retry = doc.slice(doc.indexOf('`agent.retry` is the agent reporting'), doc.indexOf('### Mapping from the trace vocabulary')); + assert.ok(retry.length > 0, 'the agent.retry paragraph is present'); + for (const type of RETRY_ERROR_TYPES) assert.ok(retry.includes(`\`${type}\``), `the doc lists ${type}`); + for (const [, type] of retry.matchAll(/`(provider\.[a-z-]+)`/g)) assert.ok(RETRY_ERROR_TYPES.includes(type), `the doc lists only the set: ${type}`); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js new file mode 100644 index 0000000000..b1aa6fd44d --- /dev/null +++ b/packages/seats/test/record-otel.test.js @@ -0,0 +1,258 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { toOtlpJson } from '../src/record/otel.js'; +import { buildSample } from '../test-support/record-sample.js'; + +const sample = async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-otel-')); + try { return await buildSample(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const spansOf = (otlp) => otlp.resourceSpans[0].scopeSpans[0].spans; +const attrOf = (holder, key) => holder.attributes.find((a) => a.key === key)?.value; +const ANY_VALUE_KEYS = new Set(['stringValue', 'boolValue', 'intValue', 'doubleValue', 'arrayValue', 'kvlistValue']); + +const checkAnyValue = (value, where) => { + const keys = Object.keys(value); + assert.equal(keys.length, 1, `${where}: one AnyValue field`); + assert.ok(ANY_VALUE_KEYS.has(keys[0]), `${where}: ${keys[0]}`); + if (keys[0] === 'intValue') assert.match(value.intValue, /^-?\d+$/, `${where}: intValue is a decimal string in OTLP/JSON`); + if (keys[0] === 'arrayValue') value.arrayValue.values.forEach((v, i) => checkAnyValue(v, `${where}[${i}]`)); + if (keys[0] === 'kvlistValue') value.kvlistValue.values.forEach((kv) => checkAnyValue(kv.value, `${where}.${kv.key}`)); +}; + +test('OTLP/JSON structure: ids, times, kinds, typed attributes', async () => { + const { entries } = await sample(); + const otlp = toOtlpJson(entries, { serviceName: 'seats-test' }); + assert.equal(otlp.resourceSpans.length, 1); + assert.deepEqual(otlp.resourceSpans[0].resource.attributes, [{ key: 'service.name', value: { stringValue: 'seats-test' } }]); + assert.deepEqual(otlp.resourceSpans[0].scopeSpans[0].scope, { name: 'seats.record', version: '1' }); + const spans = spansOf(otlp); + assert.ok(spans.length > 0); + const ids = new Set(); + for (const span of spans) { + assert.match(span.traceId, /^[0-9a-f]{32}$/); + assert.match(span.spanId, /^[0-9a-f]{16}$/); + assert.ok(!/^0+$/.test(span.traceId) && !/^0+$/.test(span.spanId)); + assert.ok(!ids.has(span.spanId), `span ids are unique: ${span.name}`); + ids.add(span.spanId); + if (span.parentSpanId !== undefined) assert.match(span.parentSpanId, /^[0-9a-f]{16}$/); + assert.equal(span.kind, 1, 'INTERNAL'); + assert.match(span.startTimeUnixNano, /^\d+$/); + assert.match(span.endTimeUnixNano, /^\d+$/, `${span.name} has an end`); + assert.ok(BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano)); + assert.ok([0, 2].includes(span.status.code)); + for (const a of span.attributes) checkAnyValue(a.value, `${span.name}.${a.key}`); + for (const event of span.events) { + assert.match(event.timeUnixNano, /^\d+$/); + for (const a of event.attributes) checkAnyValue(a.value, `${span.name}/${event.name}.${a.key}`); + } + } + // Content stays out: no message text anywhere in the export. + assert.ok(!JSON.stringify(otlp).includes('Fixed the half-cent rounding in total.js')); +}); + +test('one task is one trace; ids follow the documented derivation', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const expectedTrace = createHash('sha256').update('seats.record/trace/task/pod-checkout/TASK-101').digest('hex').slice(0, 32); + assert.deepEqual([...new Set(spans.map((s) => s.traceId))], [expectedTrace]); + const runSpan = spans.find((s) => s.name === 'seats.run' && attrOf(s, 'seats.run.id').stringValue === 'run-ana-0001'); + assert.equal(runSpan.spanId, createHash('sha256').update('seats.record/span/run-ana-0001/0').digest('hex').slice(0, 16)); + const first = entries.find((e) => e.runId === 'run-ana-0001' && e.kind === 'attempt.started'); + const attempt = spans.find((s) => s.spanId === createHash('sha256').update(`seats.record/span/run-ana-0001/${first.seq}`).digest('hex').slice(0, 16)); + assert.ok(attempt.name.startsWith('invoke_agent')); +}); + +test('parent/child links: run → attempt → tool, and decisions sit on their tool span', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const byId = new Map(spans.map((s) => [s.spanId, s])); + const runs = spans.filter((s) => s.name === 'seats.run'); + assert.equal(runs.length, 2); + for (const run of runs) assert.equal(run.parentSpanId, undefined, 'a run span is a root'); + const attempts = spans.filter((s) => s.name.startsWith('invoke_agent')); + assert.equal(attempts.length, 3, 'two attempts for Ana, one for Ben'); + for (const attempt of attempts) { + assert.equal(byId.get(attempt.parentSpanId).name, 'seats.run'); + assert.equal(attrOf(attempt, 'gen_ai.operation.name').stringValue, 'invoke_agent'); + } + const tools = spans.filter((s) => s.name.startsWith('execute_tool')); + for (const tool of tools) { + assert.ok(byId.get(tool.parentSpanId).name.startsWith('invoke_agent'), `${tool.name} is a child of an attempt`); + assert.equal(attrOf(tool, 'gen_ai.operation.name').stringValue, 'execute_tool'); + } + const resolve = tools.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + const parent = byId.get(resolve.parentSpanId); + assert.equal(attrOf(parent, 'seats.attempt.number').intValue, '2', 'the approved call ran in the retry'); + assert.equal(attrOf(resolve, 'gen_ai.tool.call.id').stringValue, 'tc-3'); + assert.equal(attrOf(resolve, 'seats.tool.surface').stringValue, 'connector'); + const decided = resolve.events.find((e) => e.name === 'permission.decided'); + assert.equal(attrOf(decided, 'seats.decided_by').stringValue, 'u-ben'); + assert.equal(attrOf(decided, 'seats.decided_by_role').stringValue, 'credential-owner'); + assert.equal(attrOf(decided, 'seats.actor.id').stringValue, 'u-ben'); + assert.equal(attrOf(parent, 'seats.agent.owner').stringValue, 'u-ana'); + // The stuck first attempt is an error with its failure kind; the run is not. + const stuck = attempts.find((s) => attrOf(s, 'seats.attempt.number').intValue === '1' && attrOf(s, 'seats.agent.owner').stringValue === 'u-ana'); + assert.equal(stuck.status.code, 2); + assert.equal(attrOf(stuck, 'error.type').stringValue, 'stuck'); + const push = tools.find((s) => s.name.startsWith('execute_tool git push')); + assert.equal(push.status.code, 2); + assert.equal(runs.every((r) => r.status.code === 0), true); +}); + +test('the export is deterministic: same entries, same bytes, in any input order', async () => { + const a = await sample(); + const b = await sample(); + const one = JSON.stringify(toOtlpJson(a.entries)); + assert.equal(one, JSON.stringify(toOtlpJson(b.entries))); + const reversedPerRun = [...a.entries.filter((e) => e.runId === 'run-ana-0001').reverse(), ...a.entries.filter((e) => e.runId === 'run-ben-0001')]; + assert.equal(JSON.stringify(toOtlpJson(reversedPerRun)), one); +}); + +test('a run without a task is its own trace; unknown kinds and newer versions are skipped', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-solo-01', actor: null, prev: null }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', data: { attempt: 1 } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'future.kind', data: {} }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'run.settled', data: { outcome: 'failed', failure: { kind: 'crashed' } } }, + { ...base, v: 2, runId: 'run-v2-0001', seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', data: {} }, + ]; + const spans = spansOf(toOtlpJson(entries)); + assert.equal(spans.length, 2); + assert.equal(spans[0].traceId, createHash('sha256').update('seats.record/trace/run/run-solo-01').digest('hex').slice(0, 32)); + assert.ok(spans.every((s) => s.events.every((e) => e.name !== 'future.kind'))); + assert.equal(spans[0].status.code, 2); + assert.equal(attrOf(spans[0], 'error.type').stringValue, 'crashed'); + assert.equal(attrOf(spans[1], 'error.type').stringValue, 'crashed', 'an unsettled last attempt takes the run\'s failure'); +}); + +test('decision.ruled joins the tool span of the request on its card, else the span current at the time; argsDigest is a tool attribute', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const resolve = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.match(attrOf(resolve, 'seats.tool.args_digest').stringValue, /^sha256:[0-9a-f]{64}$/); + const asked = resolve.events.find((e) => e.name === 'permission.requested'); + assert.equal(attrOf(asked, 'seats.asked_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(asked, 'seats.decision_card_id').stringValue, 'card-7f3'); + assert.equal(attrOf(asked, 'seats.args_digest').stringValue, attrOf(resolve, 'seats.tool.args_digest').stringValue); + // The retry ruling rode no tool call: it lands on Ana's run span, between attempts. + const anaRun = spans.find((s) => s.name === 'seats.run' && attrOf(s, 'seats.run.id').stringValue === 'run-ana-0001'); + const ruled = anaRun.events.find((e) => e.name === 'decision.ruled'); + assert.equal(attrOf(ruled, 'seats.value').stringValue, 'retry'); + assert.equal(attrOf(ruled, 'seats.decided_by_role').stringValue, 'agent-owner'); + // A ruling on a card a request carried goes to that request's tool span. + const ana = entries.filter((e) => e.runId === 'run-ana-0001'); + const settle = ana.at(-1); + const extra = [...ana.slice(0, -1), { + ...settle, kind: 'decision.ruled', data: { decisionCardId: 'card-7f3', value: 'approved', decidedBy: 'u-ben', decidedByRole: 'credential-owner' }, + }, { ...settle, seq: settle.seq + 1 }]; + const again = spansOf(toOtlpJson(extra)).find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.ok(again.events.some((e) => e.name === 'decision.ruled')); +}); + +test('actor.onBehalfOf maps beside actor.owner: on every event, the attempt span and the tool span', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-obo-0001', prev: null }; + const actor = { type: 'agent', id: 'u-agent-ana', owner: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', actor, data: { attempt: 1 } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'tool.call', actor, data: { toolCallId: 'tc-1', name: 'sentry.resolve_issue' } }, + { + ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'permission.requested', actor, + data: { toolCallId: 'tc-1', question: 'Resolve?', askedOf: { id: 'u-ben', role: 'credential-owner' } }, + }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'run.settled', actor: { type: 'system', id: 'seats' }, data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const attempt = spans.find((s) => s.name.startsWith('invoke_agent')); + assert.equal(attrOf(attempt, 'seats.agent.owner').stringValue, 'u-ana'); + assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.role').stringValue, 'requester'); + const tool = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.equal(attrOf(tool, 'seats.actor.on_behalf_of.id').stringValue, 'u-ben', 'the pair is on the action itself'); + assert.equal(attrOf(tool, 'seats.actor.on_behalf_of.role').stringValue, 'requester'); + const asked = tool.events.find((e) => e.name === 'permission.requested'); + assert.equal(attrOf(asked, 'seats.actor.owner').stringValue, 'u-ana'); + assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.role').stringValue, 'requester'); + // Without the field, no attribute appears. + const plain = spansOf(toOtlpJson(entries.map((e) => (e.actor.type === 'agent' ? { ...e, actor: { type: 'agent', id: 'u-agent-ana', owner: 'u-ana' } } : e)))); + assert.ok(!JSON.stringify(plain).includes('on_behalf_of')); +}); + +test('egress entries are events on the span current at the time', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-egr-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'egress.denied', data: { host: null, port: null, reason: 'bad_request' } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', actor: { type: 'agent', id: 'u-agent-ana' }, data: { attempt: 1 } }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'egress.allowed', data: { host: 'api.sentry.io', port: 443 } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'egress.denied', data: { host: '[redacted]', port: 443, reason: 'not_allowed' } }, + { ...base, seq: 5, ts: '2026-10-25T12:00:04.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const run = spans.find((s) => s.name === 'seats.run'); + const attempt = spans.find((s) => s.name.startsWith('invoke_agent')); + const early = run.events.find((e) => e.name === 'egress.denied'); + assert.equal(attrOf(early, 'seats.reason').stringValue, 'bad_request'); + assert.equal(attrOf(early, 'seats.host'), undefined, 'a null host is left out'); + assert.equal(attrOf(early, 'seats.actor.id').stringValue, 'seats'); + const allowed = attempt.events.find((e) => e.name === 'egress.allowed'); + assert.equal(attrOf(allowed, 'seats.host').stringValue, 'api.sentry.io'); + assert.equal(attrOf(allowed, 'seats.port').intValue, '443'); + const denied = attempt.events.find((e) => e.name === 'egress.denied'); + assert.equal(attrOf(denied, 'seats.host').stringValue, '[redacted]'); + assert.equal(attrOf(denied, 'seats.reason').stringValue, 'not_allowed'); + assert.equal(attrOf(attempt, 'error.type'), undefined, 'a denial is an event, not an error status'); +}); + +test('sandbox.applied is an event on the span current at the time', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-sbx-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'sandbox.applied', data: { sandbox: 'none', reason: 'no supported OS sandbox on win32' } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', actor: { type: 'agent', id: 'u-agent-ana' }, data: { attempt: 1 } }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'sandbox.applied', data: { sandbox: 'seatbelt' } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const none = spans.find((s) => s.name === 'seats.run').events.find((e) => e.name === 'sandbox.applied'); + assert.equal(attrOf(none, 'seats.sandbox').stringValue, 'none'); + assert.equal(attrOf(none, 'seats.reason').stringValue, 'no supported OS sandbox on win32'); + const seatbelt = spans.find((s) => s.name.startsWith('invoke_agent')).events.find((e) => e.name === 'sandbox.applied'); + assert.equal(attrOf(seatbelt, 'seats.sandbox').stringValue, 'seatbelt'); + assert.equal(attrOf(seatbelt, 'seats.reason'), undefined); +}); + +test('session.opened is an event that names the attempt\'s conversation; context usage lands on the attempt span', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-ses-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const agent = { type: 'agent', id: 'u-agent-ana' }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', actor: agent, data: { attempt: 1, adapter: 'acp' } }, + { + ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'session.opened', + data: { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1', fallback: 'session/resume answered error -32601' }, + }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'usage', actor: agent, data: { contextTokens: 900, contextSize: 128000 } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'usage', actor: agent, data: { inputTokens: 1200, outputTokens: 80, contextTokens: null } }, + { ...base, seq: 5, ts: '2026-10-25T12:00:04.000Z', kind: 'attempt.started', actor: agent, data: { attempt: 2, adapter: 'acp', sessionId: 'ses-9' } }, + { ...base, seq: 6, ts: '2026-10-25T12:00:05.000Z', kind: 'session.opened', data: { method: 'resume', sessionId: 'ses-other' } }, + { ...base, seq: 7, ts: '2026-10-25T12:00:06.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const [first, second] = spans.filter((s) => s.name.startsWith('invoke_agent')); + const opened = first.events.find((e) => e.name === 'session.opened'); + assert.equal(attrOf(opened, 'seats.method').stringValue, 'new'); + assert.equal(attrOf(opened, 'seats.requested_session_id').stringValue, 'ses-1'); + assert.equal(attrOf(opened, 'seats.fallback').stringValue, 'session/resume answered error -32601'); + assert.equal(attrOf(first, 'gen_ai.conversation.id').stringValue, 'ses-2'); + assert.equal(attrOf(first, 'seats.usage.context_tokens').intValue, '900', 'a later usage without context keeps the last known'); + assert.equal(attrOf(first, 'seats.usage.context_size').intValue, '128000'); + assert.equal(attrOf(first, 'gen_ai.usage.input_tokens').intValue, '1200'); + const seen = first.events.find((e) => e.name === 'usage'); + assert.equal(attrOf(seen, 'seats.context_tokens').intValue, '900'); + // attempt.started's own sessionId wins; one conversation id per span. + assert.deepEqual(second.attributes.filter((a) => a.key === 'gen_ai.conversation.id').map((a) => a.value.stringValue), ['ses-9']); + assert.equal(attrOf(second, 'seats.usage.context_tokens'), undefined); +}); diff --git a/packages/seats/test/record-redact-stream.test.js b/packages/seats/test/record-redact-stream.test.js new file mode 100644 index 0000000000..ba178a30c3 --- /dev/null +++ b/packages/seats/test/record-redact-stream.test.js @@ -0,0 +1,258 @@ +/** + * The gaps the security gate on #2701 found in the store's redaction, each + * closed with a test that fails on the code it replaced: + * + * 1. a value split across streamed deltas matched none of them; + * 2. a value under eight characters was granted and then not scrubbed; + * 3. the runtime token and a RunFailure's detail never met the scrub. + * + * "Stored text" below is every chunk text the record holds for one stream, + * joined in order: what a reader reconstructs the agent's output from. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { superviseRun } from '../src/supervisor.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { RunFailure } from '../src/failures.js'; + +const CANARY = 'sk-test-CANARY-7f3a91c2d4e5b6a8'; +const AGENT = { type: 'agent', id: 'u-agent-stream', owner: 'u-owner-stream' }; +const OTHER = { type: 'agent', id: 'u-agent-other', owner: 'u-owner-stream' }; +const FAST = { stallMs: 0, timeoutMs: 0, cancelGraceMs: 40 }; + +const withStore = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-stream-')); + try { return await fn(createTraceStore({ root }), root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const allBytes = (root) => readdirSync(root, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()) + .map((d) => readFileSync(join(d.parentPath ?? d.path, d.name), 'utf8')) + .join('\n'); +const storedText = (store, runId, kind, actorId) => store.readRun(runId) + .filter((e) => e.kind === kind && (actorId === undefined || e.actor?.id === actorId)) + .map((e) => e.data.text) + .join(''); + +/** The checks every split case shares: not in the bytes, not in the joined text, and the marker present. */ +const assertScrubbed = (store, root, runId, streams) => { + assert.equal(allBytes(root).includes(CANARY), false, 'the value is in the record bytes'); + for (const { kind, actorId, expected } of streams) { + const text = storedText(store, runId, kind, actorId); + assert.equal(text.includes(CANARY), false, `the joined ${kind} text reconstructs the value: ${text}`); + assert.equal(text, expected, `the ${kind} stream lost or changed text`); + assert.ok(text.includes('[redacted]'), 'positive control: the scrub replaced the value'); + } + assert.equal(store.verifyRun(runId).ok, true, 'held text is written through the chain like any other line'); +}; + +const splits = (n) => Array.from({ length: n - 1 }, (_, i) => i + 1); + +test('rhea\'s case: a value split across two message deltas is scrubbed', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-split-rhea' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: 'the key is sk-test-CANA' }); + run.emit('agent.message_chunk', { text: 'RY-7f3a91c2d4e5b6a8 end' }); + const texts = store.readRun('run-split-rhea').map((e) => e.data.text); + assert.deepEqual(texts, ['the key is ', '[redacted] end']); + assert.equal(allBytes(root).includes(CANARY), false); +})); + +for (const kind of ['agent.message_chunk', 'agent.thought_chunk']) { + test(`${kind}: a value split in two, at every split point, never reaches the record`, () => withStore(async (store, root) => { + for (const at of splits(CANARY.length)) { + const runId = `run-two-${kind.length}-${at}`; + const run = store.startRun({ runId, actor: AGENT }); + run.redact([CANARY]); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit(kind, { text: `the key is ${CANARY.slice(0, at)}` }); + run.emit(kind, { text: `${CANARY.slice(at)} end` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [{ kind, expected: 'the key is [redacted] end' }]); + } + })); +} + +test('a value split three ways, at every pair of split points, never reaches the record', () => withStore(async (store, root) => { + for (const a of splits(CANARY.length)) { + for (let b = a + 1; b < CANARY.length; b += 1) { + const runId = `run-three-${a}-${b}`; + const run = store.startRun({ runId }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `key ${CANARY.slice(0, a)}` }); + run.emit('agent.message_chunk', { text: CANARY.slice(a, b) }); + run.emit('agent.message_chunk', { text: `${CANARY.slice(b)}!` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [{ kind: 'agent.message_chunk', expected: 'key [redacted]!' }]); + } + } +})); + +test('two streams interleaved, each splitting the value, are scrubbed apart', () => withStore(async (store, root) => { + for (const at of splits(CANARY.length)) { + // Message vs thought, and the same kind from two actors: four streams, one line each in turn. + const runId = `run-mix-${at}`; + const run = store.startRun({ runId, actor: AGENT }); + run.redact([CANARY]); + const head = `a ${CANARY.slice(0, at)}`; + const tail = `${CANARY.slice(at)} z`; + run.emit('agent.message_chunk', { text: head }); + run.emit('agent.thought_chunk', { text: head }); + run.emit('agent.message_chunk', { text: head }, { actor: OTHER }); + run.emit('usage', { inputTokens: 1 }); // a non-stream line between the halves releases nothing + run.emit('agent.thought_chunk', { text: tail }); + run.emit('agent.message_chunk', { text: tail }, { actor: OTHER }); + run.emit('agent.message_chunk', { text: tail }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [ + { kind: 'agent.message_chunk', actorId: AGENT.id, expected: 'a [redacted] z' }, + { kind: 'agent.message_chunk', actorId: OTHER.id, expected: 'a [redacted] z' }, + { kind: 'agent.thought_chunk', actorId: AGENT.id, expected: 'a [redacted] z' }, + ]); + } +})); + +test('a stream that ends mid-value writes the marker before settle, never the prefix', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-held-settle' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `cut off at ${CANARY.slice(0, 12)}` }); + run.emit('run.settled', { outcome: 'failed', failure: { kind: 'exited', detail: 'gone' }, durationMs: 1, attempts: 1 }); + const events = store.readRun('run-held-settle'); + assert.deepEqual(events.map((e) => [e.kind, e.data.text]), [ + ['agent.message_chunk', 'cut off at '], + ['agent.message_chunk', '[redacted]'], + ['run.settled', undefined], + ]); + assert.equal(allBytes(root).includes(CANARY.slice(0, 12)), false); +})); + +// The security gate's probe: all but the last character, then the run ends. +test('a value cut one character short never reaches the record', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-held-short' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `key: ${CANARY.slice(0, -1)}` }); + run.emit('liveness.recovery', { action: 'kill', attempt: 1 }); + run.emit('run.settled', { outcome: 'failed', failure: { kind: 'exited', detail: 'killed' }, durationMs: 1, attempts: 1 }); + assert.deepEqual(storedText(store, 'run-held-short', 'agent.message_chunk', undefined), 'key: [redacted]'); + assert.equal(allBytes(root).includes(CANARY.slice(0, -1)), false); + assert.equal(allBytes(root).includes(CANARY.slice(0, 8)), false); +})); + +// The value is registered as the run's runtime token, the one the supervisor itself hands the store. +test('a supervised run killed mid-value writes the marker for its held tail, before the settle and the retry', () => withStore(async (store, root) => { + let spawns = 0; + const adapter = { + name: 'split-then-hang', + spawn(prompt, ctx) { + spawns += 1; + ctx.trace.emit('agent.message_chunk', { text: `try ${spawns}: ${CANARY.slice(0, 10)}` }); + // Ignores the abort, so the supervisor's grace timer kills it. + return new Promise(() => {}); + }, + }; + const ctrl = new AbortController(); + setTimeout(() => ctrl.abort(), 30); + const out = await superviseRun({ + adapter, prompt: 'p', store, ctx: { signal: ctrl.signal, runtimeToken: CANARY }, policy: FAST, + }); + assert.equal(out.outcome, 'cancelled'); + const events = store.readRun(out.runId); + const settledAt = events.findIndex((e) => e.kind === 'run.settled'); + const chunks = events.filter((e) => e.kind === 'agent.message_chunk'); + assert.deepEqual(chunks.map((e) => e.data.text), ['try 1: ', '[redacted]']); + assert.ok(chunks.every((e) => e.seq < events[settledAt].seq), 'the marker landed before run.settled'); + assert.equal(allBytes(root).includes(CANARY.slice(0, 10)), false, 'no prefix of the value on disk'); + + // On retry, the first attempt's tail lands before the second attempt starts, + // not glued to the second's text. (After the retry line: an adapter may still + // stream during a cancel's grace window, so recovery lines do not release.) + let tries = 0; + const retrying = { + name: 'split-then-exit', + async spawn(prompt, ctx) { + tries += 1; + ctx.trace.emit('agent.message_chunk', { text: `try ${tries}: ${CANARY.slice(0, 10)}` }); + if (tries === 1) throw new RunFailure('exited', 'the CLI died'); + return {}; + }, + }; + const again = await superviseRun({ + adapter: retrying, prompt: 'p', store, ctx: { runtimeToken: CANARY }, policy: { ...FAST, retry: { max: 1 } }, + }); + assert.equal(again.outcome, 'ok'); + assert.deepEqual(store.readRun(again.runId).map((e) => e.kind === 'agent.message_chunk' ? e.data.text : e.kind), [ + 'attempt.started', 'try 1: ', 'liveness.recovery', '[redacted]', + 'attempt.started', 'try 2: ', '[redacted]', 'run.settled', + ]); + assert.equal(allBytes(root).includes(CANARY.slice(0, 10)), false); +})); + +test('the store scrubs a registered value of any length', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-short-1' }); + run.redact(['abc123']); + run.emit('agent.message_chunk', { text: 'value abc123 here' }); + run.emit('tool.update', { toolCallId: 't', title: 'abc123' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + const bytes = allBytes(root); + assert.equal(bytes.includes('abc123'), false, 'a six-character value reached the record'); + assert.equal(storedText(store, 'run-short-1', 'agent.message_chunk'), 'value [redacted] here'); +})); + +test('the valet refuses a credential under eight characters as credential_denied, before any HOME', async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-short-')); + try { + const events = []; + const trace = { emit: (type, data) => events.push({ type, data }), redact: () => assert.fail('a refused value was registered') }; + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'SHORT_KEY', from: 'resolver' }] }, resolveSecret: async () => 'abc123', root, trace, + }), (error) => { + assert.ok(error instanceof RunFailure); + assert.equal(error.kind, 'credential_denied'); + assert.match(error.message, /SHORT_KEY resolved to a value shorter than 8 characters/); + assert.equal(error.message.includes('abc123'), false); + return true; + }); + assert.deepEqual(events, [{ type: 'credential.denied', data: { name: 'SHORT_KEY', reason: 'too_short' } }]); + assert.deepEqual(readdirSync(root), [], 'the refused prepare removed its HOME'); + // Positive control: eight characters is enough. + const ok = await prepareRunEnv({ + grant: { credentials: [{ name: 'SHORT_KEY', from: 'resolver' }] }, resolveSecret: async () => 'abcd1234', root, trace: { emit() {} }, + }); + assert.equal(ok.env.SHORT_KEY, 'abcd1234'); + await ok.cleanup(); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +const TOKEN = 'cm_agent_runtime-token-5e6f7a8b9c0d'; + +test('the run\'s runtime token is scrubbed from agent text', () => withStore(async (store, root) => { + const adapter = { + name: 'token-echo', + async spawn(prompt, ctx) { + ctx.trace.emit('agent.message_chunk', { text: `my token is ${ctx.runtimeToken}.` }); + return {}; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { runtimeToken: TOKEN }, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.equal(allBytes(root).includes(TOKEN), false, 'the runtime token reached the record'); + assert.equal(storedText(store, out.runId, 'agent.message_chunk'), 'my token is [redacted].'); +})); + +test('a RunFailure\'s detail is scrubbed of the spawn\'s credentials before settle', () => withStore(async (store) => { + // A run with no redact hook: only failureData stands between the detail and the line. + const bare = { + startRun: (meta) => { + const { redact, ...run } = store.startRun(meta); + return run; + }, + }; + const adapter = { name: 'leaky-failure', async spawn() { throw new RunFailure('exited', `auth failed for ${TOKEN}`); } }; + const out = await superviseRun({ adapter, prompt: 'p', store: bare, ctx: { runtimeToken: TOKEN }, policy: FAST }); + assert.equal(out.outcome, 'failed'); + const [settled] = store.readRun(out.runId).filter((e) => e.kind === 'run.settled'); + assert.deepEqual(settled.data.failure, { kind: 'exited', detail: 'auth failed for [redacted]' }); +})); diff --git a/packages/seats/test/record-redact-supervised.test.js b/packages/seats/test/record-redact-supervised.test.js new file mode 100644 index 0000000000..5ec6d46f0b --- /dev/null +++ b/packages/seats/test/record-redact-supervised.test.js @@ -0,0 +1,186 @@ +/** + * A credential value the valet resolves for a supervised run never reaches + * the record, even when the agent repeats it. The value travels the real + * path: prepareRunEnv is called with the spawn ctx the supervisor hands the + * adapter, so it registers the value through the supervisor's trace wrapper, + * and the store scrubs it before a line is hashed or written. + * + * Everything is local and keyless: the fake provider answers on 127.0.0.1 and + * the "credential" is a fake value. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { toOtlpJson } from '../src/record/otel.js'; +import { runFailure } from '../src/failures.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +const CANARY = 'sk-test-CANARY-7f3a91c2d4e5b6a8'; +const KEY_NAME = 'FAKE_PROVIDER_KEY'; +const GRANT = { credentials: [{ name: KEY_NAME, from: 'resolver' }], tools: { allow: ['read'], ask: ['fetch'] } }; +const AGENT = { type: 'agent', id: 'u-agent-redact', owner: 'u-owner-redact', name: 'Redact test agent' }; +const OWNER = { id: 'u-owner-redact', role: 'agent-owner' }; +// The kinds the adapter writes itself, naming its actor. permission.* are the valet's. +const AGENT_KINDS = new Set(['agent.message_chunk', 'tool.call', 'tool.update', 'result.reported']); + +// The store's redaction is exact-match on known values (docs/record-v1.md §3, +// rule 1): a value the agent transforms, such as base64, is not claimed and +// is not asserted here. The canary is all unreserved characters, so its +// URL-encoded and JSON-escaped forms are the value itself; the check names +// them anyway so a change of canary cannot quietly drop them. +const FORMS = Object.freeze({ + raw: CANARY, + urlEncoded: encodeURIComponent(CANARY), + jsonEscaped: JSON.stringify(CANARY).slice(1, -1), +}); +const assertAbsent = (text, where) => { + for (const [form, value] of Object.entries(FORMS)) { + assert.equal(text.includes(value), false, `the ${form} credential value appears in ${where}`); + } +}; + +const allFiles = (dir) => readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()) + .map((d) => join(d.parentPath ?? d.path, d.name)); + +/** The text an OpenAI-style SSE stream carried, one string per content delta. */ +const sseDeltas = async (response) => { + const body = await response.text(); + return body.split('\n') + .filter((line) => line.startsWith('data: ') && line !== 'data: [DONE]') + .map((line) => JSON.parse(line.slice(6)).choices?.[0]?.delta?.content) + .filter((text) => typeof text === 'string' && text.length > 0); +}; + +/** + * An adapter that does what a CLI adapter does with a granted key: asks the + * valet for its env, calls the provider with it, and streams what comes back. + * The provider is scripted to echo the key, so the key lands in agent text, a + * tool result, the question the valet's permission policy records, and the + * final result. + */ +const echoingAdapter = ({ baseURL, envRoot, fail, seen }) => ({ + name: 'echo-fake', + providerKeyEnv: KEY_NAME, + async spawn(prompt, ctx) { + const prepared = await prepareRunEnv({ + grant: GRANT, resolveSecret: async () => CANARY, root: envRoot, ctx, require: [KEY_NAME], + }); + try { + const key = prepared.env[KEY_NAME]; + seen.key = key; + const as = { actor: AGENT }; + ctx.trace.emit('turn.prompt_sent', { chars: prompt.length }); + const response = await fetch(`${baseURL}/chat/completions`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ model: 'fake-model', stream: true, messages: [{ role: 'user', content: `[[reply:the key is ${key}]]` }] }), + }); + const deltas = await sseDeltas(response); + seen.reply = deltas.join(''); + for (const text of deltas) ctx.trace.emit('agent.message_chunk', { text }, as); + ctx.trace.emit('tool.call', { + toolCallId: 'tc-1', title: `printenv ${KEY_NAME}`, kind: 'execute', status: 'pending', surface: 'native', + }, as); + ctx.trace.emit('tool.update', { toolCallId: 'tc-1', status: 'completed', title: `${KEY_NAME}=${key}` }, as); + const decide = createPermissionPolicy(GRANT, { + trace: ctx.trace, askedOf: OWNER, approve: async () => ({ approved: true, by: OWNER.id, role: OWNER.role }), + }); + const answer = await decide({ + toolCall: { toolCallId: 'tc-2', title: `Send ${key} to the deploy hook?`, kind: 'fetch', rawInput: { token: key } }, + options: [{ optionId: 'allow', kind: 'allow_once' }, { optionId: 'reject', kind: 'reject_once' }], + }); + seen.answer = answer; + if (fail) throw runFailure('exited', `the CLI exited after printing ${key}`); + ctx.trace.emit('agent.message_chunk', { text: `Done. I used ${key}.` }, as); + ctx.trace.emit('result.reported', { status: 'done', refs: [{ type: 'note', id: `used-${key}` }] }, as); + return { text: seen.reply }; + } finally { + await prepared.cleanup(); + } + }, +}); + +const withHarness = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-sup-')); + const fake = await startFakeProvider(); + try { + const traceRoot = join(root, 'trace'); + return await fn({ fake, root, traceRoot, store: createTraceStore({ root: traceRoot }) }); + } finally { + await fake.close(); + rmSync(root, { recursive: true, force: true }); + } +}; + +for (const fail of [false, true]) { + test(`a granted credential the agent echoes never reaches the record or its export (${fail ? 'failed' : 'ok'} run)`, () => withHarness(async ({ + fake, root, traceRoot, store, + }) => { + const seen = {}; + const out = await superviseRun({ + adapter: echoingAdapter({ baseURL: fake.baseURL, envRoot: root, fail, seen }), + prompt: 'deploy it', + store, + policy: { stallMs: 0, timeoutMs: 0 }, + }); + + // Positive controls: the value really reached the run and the agent really repeated it. + assert.equal(seen.key, CANARY, 'the valet resolved the planted value into the run env'); + assert.ok(seen.reply.includes(CANARY), 'the provider echoed the value back as agent text'); + assert.deepEqual(fake.requests.map((r) => r.authorization), ['present']); + assert.equal(out.outcome, fail ? 'failed' : 'ok'); + if (fail) assert.ok(out.failure.detail.includes(CANARY), 'the failure the run settled on quoted the value'); + + // Every byte the store wrote: the run file and the index. + const files = allFiles(traceRoot); + assert.deepEqual(files.map((f) => f.slice(traceRoot.length + 1)).sort(), ['index.jsonl', `runs/${out.runId}.jsonl`]); + const bytes = files.map((f) => readFileSync(f, 'utf8')).join('\n'); + assertAbsent(bytes, 'the record files'); + + const entries = store.readRun(out.runId); + // The export reads what the store hands back, so it is checked on its own, + // before anything else about those entries. + const otlp = JSON.stringify(toOtlpJson(entries)); + assertAbsent(otlp, 'the OTLP export'); + assert.ok(otlp.includes('[redacted]'), 'positive control: a scrubbed field reached the export'); + assert.ok(otlp.includes('seats.question'), 'positive control: the permission question reached the export'); + assert.ok(otlp.includes('Send [redacted] to the deploy hook?'), 'the exported question is the scrubbed one'); + + const byKind = (kind) => entries.filter((e) => e.kind === kind); + // Each place the value was planted holds the marker instead, so its absence above is a scrub, not a drop. + assert.ok(byKind('agent.message_chunk').some((e) => e.data.text.includes('[redacted]')), 'agent text was scrubbed'); + assert.equal(byKind('tool.update')[0].data.title, `${KEY_NAME}=[redacted]`, 'the tool result was scrubbed'); + const [asked] = byKind('permission.requested'); + assert.equal(asked.data.question, 'Send [redacted] to the deploy hook?', 'the permission question was scrubbed'); + assert.deepEqual(asked.data.askedOf, OWNER); + assert.match(asked.data.argsDigest, /^sha256:[0-9a-f]{64}$/, 'the arguments are recorded as a digest only'); + assert.deepEqual(seen.answer, { outcome: 'selected', optionId: 'allow' }); + const [settled] = byKind('run.settled'); + if (fail) { + assert.equal(settled.data.failure.detail, 'the CLI exited after printing [redacted]', 'the final failure was scrubbed'); + const indexRow = readFileSync(join(traceRoot, 'index.jsonl'), 'utf8').split('\n').filter(Boolean).map(JSON.parse).find((r) => r.outcome); + assert.equal(indexRow.failure.detail, 'the CLI exited after printing [redacted]', 'the index row was scrubbed'); + } else { + assert.equal(byKind('result.reported')[0].data.refs[0].id, 'used-[redacted]', 'the final result was scrubbed'); + assert.ok(byKind('agent.message_chunk').some((e) => e.data.text === 'Done. I used [redacted].')); + } + assert.deepEqual(byKind('credential.granted')[0].data, { names: [KEY_NAME] }, 'the valet recorded the name through the wrapper'); + + // Scrubbed before hashing: the chain over the scrubbed lines is intact. + const verified = store.verifyRun(out.runId); + assert.equal(verified.ok, true, JSON.stringify(verified.errors)); + assert.equal(verified.entries, entries.length); + + // The adapter's own entries carry the actor it named. + const agentEntries = entries.filter((e) => AGENT_KINDS.has(e.kind)); + assert.ok(agentEntries.length >= (fail ? 3 : 5), `${agentEntries.length} adapter entries`); + for (const entry of agentEntries) assert.deepEqual(entry.actor, AGENT, `actor on ${entry.kind} #${entry.seq}`); + })); +} diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js new file mode 100644 index 0000000000..a1b3fd2dc7 --- /dev/null +++ b/packages/seats/test/record-store.test.js @@ -0,0 +1,195 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { appendFileSync, chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { verifyRecord } from '../src/record/format.js'; + +const AGENT = { type: 'agent', id: 'u-agent-ana', owner: 'u-ana', name: "Ana's agent" }; +const withStore = (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-')); + let tick = Date.UTC(2026, 9, 25, 12, 0, 0); + try { return fn(createTraceStore({ root, now: () => new Date((tick += 1000)) }), root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const fileOf = (root, runId) => join(root, 'runs', `${runId}.jsonl`); +const grantMeta = (grantId, extra = {}) => ({ + grantId, scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], audience: ['u-ana'] }, issuedBy: 'u-ana', ...extra, +}); + +test('readRun keeps the trace shape (type, data, runId, seq, ts) and adds the envelope', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0001', adapter: 'fake', taskId: 'TASK-9', podId: 'pod-a', actor: AGENT }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('agent.message_chunk', { text: 'hi' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + const events = store.readRun('run-st-0001'); + assert.deepEqual(events.map((e) => e.type), ['run.started', 'agent.message_chunk', 'run.settled']); + assert.deepEqual(events.map((e) => e.kind), ['attempt.started', 'agent.message_chunk', 'run.settled']); + assert.deepEqual(events.map((e) => e.seq), [1, 2, 3]); + for (const event of events) { + assert.equal(event.runId, 'run-st-0001'); + assert.equal(event.taskId, 'TASK-9'); + assert.equal(event.podId, 'pod-a'); + assert.equal(typeof event.ts, 'string'); + assert.equal(event.v, 1); + } + assert.deepEqual(events[0].actor, AGENT); + assert.deepEqual(events[0].data, { adapter: 'fake', attempt: 1 }); + const row = store.listRuns().find((r) => r.runId === 'run-st-0001'); + assert.equal(row.outcome, 'ok'); + assert.equal(row.taskId, 'TASK-9'); + assert.equal(row.entries, 3); + assert.equal(store.verifyRun('run-st-0001').ok, true); +})); + +test('startRun refuses a taskId without a podId, and a display name as actor id', () => withStore((store) => { + assert.throws(() => store.startRun({ runId: 'run-st-0002', taskId: 'TASK-9' }), /needs a podId/); + assert.throws(() => store.startRun({ runId: 'run-st-0003', actor: { type: 'agent', id: 'Ana agent' } }), /actor.id/); +})); + +test('emit: unknown types throw, and nothing is written after run.settled', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0004' }); + assert.throws(() => run.emit('made.up', {}), /unknown trace event type/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.equal(run.emit('usage', {}), null); + assert.equal(store.readRun('run-st-0004').length, 1); +})); + +test('files are 0600 and directories 0700', () => withStore((store, root) => { + store.startRun({ runId: 'run-st-0005' }).emit('usage', {}); + assert.equal(statSync(join(root, 'runs')).mode & 0o777, 0o700); + assert.equal(statSync(fileOf(root, 'run-st-0005')).mode & 0o777, 0o600); + assert.equal(statSync(join(root, 'index.jsonl')).mode & 0o777, 0o600); +})); + +test('verifyRun: a torn last line is reported', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0006' }); + run.emit('usage', {}); + run.emit('usage', {}); + appendFileSync(fileOf(root, 'run-st-0006'), '{"v":1,"schema":"seats.rec'); + const result = store.verifyRun('run-st-0006'); + assert.equal(result.ok, false); + assert.deepEqual(result.errors, [{ seq: 3, reason: 'torn' }]); + assert.equal(store.readRun('run-st-0006').length, 2, 'readRun still returns the whole lines'); +})); + +test('verifyRun: the index head catches an edited or dropped last line', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0007' }); + run.emit('usage', { inputTokens: 1 }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + const path = fileOf(root, 'run-st-0007'); + const original = readFileSync(path, 'utf8'); + writeFileSync(path, original.replace('"durationMs":1', '"durationMs":2')); + assert.equal(verifyRecord(readFileSync(path, 'utf8')).ok, true, 'the chain alone cannot see an edit to the last line'); + assert.deepEqual(store.verifyRun('run-st-0007').errors, [{ seq: 2, reason: 'head_mismatch' }]); + writeFileSync(path, original.split('\n').slice(0, 1).join('\n').concat('\n')); + assert.deepEqual(store.verifyRun('run-st-0007').errors.map((e) => e.reason), ['count_mismatch']); +})); + +test('a failed append breaks the run: later emits throw instead of chaining past an unknown tail', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0008' }); + run.emit('usage', {}); + chmodSync(fileOf(root, 'run-st-0008'), 0o400); + try { + assert.throws(() => run.emit('usage', {})); + chmodSync(fileOf(root, 'run-st-0008'), 0o600); + assert.throws(() => run.emit('usage', {}), /broken/); + } finally { chmodSync(fileOf(root, 'run-st-0008'), 0o600); } +})); + +test('task.done ends every grant bound to the task, in one grant.revoked written by the board', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0009', taskId: 'TASK-9', podId: 'pod-a', actor: AGENT }); + run.emit('grant.issued', { ...grantMeta('g-bound-1'), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'task.done' } }); + run.emit('grant.issued', { ...grantMeta('g-bound-2'), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'task.done' } }); + run.emit('grant.issued', { + ...grantMeta('g-timed', { expiresAt: '2026-10-25T23:00:00Z' }), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'expiresAt' }, + }); + run.emit('grant.issued', grantMeta('g-free')); + run.emit('task.done', { doneBy: 'u-ben' }); + const events = store.readRun('run-st-0009'); + const [done, revoked] = events.slice(-2); + assert.equal(done.kind, 'task.done'); + assert.deepEqual(done.actor, { type: 'system', id: 'board' }); + assert.equal(revoked.kind, 'grant.revoked'); + assert.deepEqual(revoked.data, { grantIds: ['g-bound-1', 'g-bound-2'], reason: 'task.done' }); + assert.deepEqual(revoked.actor, { type: 'system', id: 'board' }); + assert.deepEqual(run.openGrants(), ['g-timed', 'g-free']); +})); + +test('revokeGrant: a lineage is one cascade entry; a lone grant is manual', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0010' }); + run.emit('grant.issued', grantMeta('g-root')); + run.emit('grant.issued', grantMeta('g-child', { parentGrantId: 'g-root' })); + run.emit('grant.issued', grantMeta('g-grandchild', { parentGrantId: 'g-child' })); + run.emit('grant.issued', grantMeta('g-lone')); + const cascade = run.revokeGrant('g-root', { actor: { type: 'human', id: 'u-ben' } }); + assert.deepEqual(cascade.data, { grantIds: ['g-root', 'g-child', 'g-grandchild'], reason: 'cascade' }); + assert.deepEqual(cascade.actor, { type: 'human', id: 'u-ben' }); + assert.deepEqual(run.revokeGrant('g-lone').data, { grantIds: ['g-lone'], reason: 'manual' }); + assert.equal(run.revokeGrant('g-lone'), null, 'an ended grant is not revoked twice'); + assert.deepEqual(run.openGrants(), []); +})); + +test('expireGrants records every grant past its expiresAt as expired', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0011' }); + run.emit('grant.issued', grantMeta('g-old', { expiresAt: '2026-10-25T12:00:01Z' })); + run.emit('grant.issued', grantMeta('g-new', { expiresAt: '2026-10-26T00:00:00Z' })); + const entry = run.expireGrants(new Date('2026-10-25T13:00:00Z')); + assert.deepEqual(entry.data, { grantIds: ['g-old'], reason: 'expired' }); + assert.equal(run.expireGrants(new Date('2026-10-25T13:00:00Z')), null); +})); + +test('redact: known values never reach a line, even inside agent text', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0012' }); + run.redact(['sk-secret-0123456789']); + run.emit('agent.message_chunk', { text: 'my key is sk-secret-0123456789, ok' }); + run.emit('tool.call', { toolCallId: 't', title: 'curl -H "x: sk-secret-0123456789"' }); + const raw = readFileSync(fileOf(root, 'run-st-0012'), 'utf8'); + assert.ok(!raw.includes('sk-secret-0123456789')); + assert.ok(raw.includes('[redacted]')); + assert.equal(store.verifyRun('run-st-0012').ok, true, 'scrubbing happens before hashing'); +})); + +test('the store writes actor.onBehalfOf through emit, and refuses it on a non-agent run actor', () => withStore((store, root) => { + const actor = { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester' } }; + const run = store.startRun({ runId: 'run-st-obo1', adapter: 'fake', taskId: 'TASK-9', podId: 'pod-a', actor }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('tool.call', { toolCallId: 'tc-1', title: 'resolve', surface: 'connector' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + const events = store.readRun('run-st-obo1'); + assert.deepEqual(events[1].actor, actor); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-obo1'), 'utf8')).errors, []); + assert.throws(() => store.startRun({ runId: 'run-st-obo2', actor: { type: 'human', id: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } } }), /agent actors only/); + assert.throws(() => store.startRun({ runId: 'run-st-obo3', actor: { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'boss' } } }), /onBehalfOf.role/); +})); + +test('the store records the egress proxy\'s trace types', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-egr1', actor: AGENT }); + run.emit('egress.allowed', { host: 'api.sentry.io', port: 443 }); + run.emit('egress.denied', { host: null, port: null, reason: 'bad_request' }); + run.emit('egress.denied', { host: '[redacted]', port: 443, reason: 'not_allowed' }); + assert.throws(() => run.emit('egress.denied', { host: 'x.io', port: 443, reason: 'nope' }), /egress.denied.reason/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-egr1').map((e) => e.kind), ['egress.allowed', 'egress.denied', 'egress.denied', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-egr1'), 'utf8')).errors, []); +})); + +test('the store records sandbox.applied', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-sbx1', actor: AGENT }); + run.emit('sandbox.applied', { sandbox: 'none', reason: 'no supported OS sandbox on win32' }); + assert.throws(() => run.emit('sandbox.applied', { sandbox: 'bwrap', reason: 'x' }), /only when sandbox is none/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-sbx1').map((e) => e.kind), ['sandbox.applied', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-sbx1'), 'utf8')).errors, []); +})); + +test('the store records session.opened and context usage', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-ses1', actor: AGENT }); + run.emit('run.started', { adapter: 'acp', attempt: 1 }); + run.emit('session.opened', { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1', fallback: 'session/resume answered error -32601' }); + run.emit('usage', { inputTokens: null, outputTokens: null, totalTokens: null, cost: null, contextTokens: 900, contextSize: 128000 }); + assert.throws(() => run.emit('session.opened', { method: 'new', sessionId: 'ses-3', requestedSessionId: 'ses-1' }), /come together/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-ses1').map((e) => e.kind), ['attempt.started', 'session.opened', 'usage', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-ses1'), 'utf8')).errors, []); +})); diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js new file mode 100644 index 0000000000..bd649809f5 --- /dev/null +++ b/packages/seats/test/record-valet.test.js @@ -0,0 +1,280 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy, MAX_REFUSED_ANSWERS } from '../src/valet/permissions.js'; +import { argsDigestOf } from '../src/record/format.js'; + +const FROM_ENV = 'sk-env-planted-0a1b2c3d4e5f'; +const FROM_FILE = 'sk-file-planted-6a7b8c9d0e1f'; +const FROM_RESOLVER = 'sk-resolver-planted-2f3e4d5c'; +const UNGRANTED = 'sk-ungranted-planted-9z8y7x6w'; +const OPTIONS = [ + { optionId: 'allow', kind: 'allow_once' }, { optionId: 'always', kind: 'allow_always' }, { optionId: 'reject', kind: 'reject_once' }, +]; + +const withRoot = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-valet-')); + try { return await fn(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const everyByte = (dir) => readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()).map((d) => readFileSync(join(d.parentPath ?? d.path, d.name), 'utf8')).join('\n'); + +test('no secret values in the record: env, file and resolver values, even echoed by the agent', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0001', taskId: 'TASK-1', podId: 'pod-a', actor: { type: 'agent', id: 'u-agent-1', owner: 'u-ana' } }); + const secretFile = join(root, 'token.txt'); + writeFileSync(secretFile, `${FROM_FILE}\n`, { mode: 0o600 }); + const grant = { + credentials: [ + { name: 'OPENAI_API_KEY', from: 'env' }, { name: 'FILE_TOKEN', from: 'file', ref: secretFile }, { name: 'BROKER_TOKEN', from: 'resolver' }, + ], + tools: { allow: ['read'], ask: ['fetch'] }, + }; + const env = await prepareRunEnv({ + grant, + baseEnv: { OPENAI_API_KEY: FROM_ENV, ANTHROPIC_API_KEY: UNGRANTED }, + resolveSecret: async () => FROM_RESOLVER, + root, + ctx: { trace: run }, + grantMeta: { + grantId: 'g-1', scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], audience: ['u-ana'] }, issuedBy: 'u-ana', taskId: 'TASK-1', podId: 'pod-a', + }, + }); + // Positive control: the values really reached the run, so their absence below means something. + assert.equal(env.env.OPENAI_API_KEY, FROM_ENV); + assert.equal(env.env.FILE_TOKEN, FROM_FILE); + assert.equal(env.env.BROKER_TOKEN, FROM_RESOLVER); + const decide = createPermissionPolicy(grant, { trace: run, approve: async () => ({ approved: true, by: 'u-ben', role: 'credential-owner' }) }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('agent.message_chunk', { text: `env ${FROM_ENV} file ${FROM_FILE} resolver ${FROM_RESOLVER}` }); + run.emit('tool.call', { toolCallId: 'tc-1', title: `curl -H "Authorization: Bearer ${FROM_RESOLVER}"`, kind: 'fetch' }); + await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch', kind: 'fetch' }, options: OPTIONS }); + run.emit('usage', { note: `${FROM_FILE}` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 3, attempts: 1 }); + await env.cleanup(); + + const bytes = everyByte(join(root, 'trace')); + assert.ok(bytes.includes('agent.message_chunk'), 'positive control: the agent text was written'); + for (const value of [FROM_ENV, FROM_FILE, FROM_RESOLVER, UNGRANTED]) assert.ok(!bytes.includes(value), `${value.slice(0, 12)}… reached the trace`); + assert.equal((bytes.match(/\[redacted\]/g) || []).length, 5); + const kinds = store.readRun('run-va-0001').map((e) => e.kind); + assert.deepEqual(kinds.slice(0, 2), ['grant.issued', 'credential.granted']); + assert.deepEqual(store.readRun('run-va-0001')[1].data, { names: ['OPENAI_API_KEY', 'FILE_TOKEN', 'BROKER_TOKEN'] }); + assert.equal(store.verifyRun('run-va-0001').ok, true); +})); + +test('grant.issued from grantMeta: bound to the task, ending on task.done by default', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0002', taskId: 'TASK-2', podId: 'pod-a' }); + const env = await prepareRunEnv({ + grant: { credentials: [] }, root, trace: run, + grantMeta: { + grantId: 'g-2', parentGrantId: 'g-root', issuedBy: 'u-ben', taskId: 'TASK-2', podId: 'pod-a', + scope: { connectionId: 'conn-ben', tools: ['github.get_pull_request'], writeMode: 'read', budget: { calls: 3 }, audience: ['u-ben'] }, + target: { kind: 'seat', id: 'u-agent-ben' }, + }, + }); + await env.cleanup(); + assert.equal(env.grantId, 'g-2'); + const [issued] = store.readRun('run-va-0002'); + assert.equal(issued.kind, 'grant.issued'); + assert.deepEqual(issued.data.binding, { taskId: 'TASK-2', podId: 'pod-a', endsOn: 'task.done' }); + assert.equal(issued.data.parentGrantId, 'g-root'); + assert.deepEqual(issued.actor, { type: 'human', id: 'u-ben' }); + run.emit('task.done', {}); + assert.deepEqual(store.readRun('run-va-0002').at(-1).data, { grantIds: ['g-2'], reason: 'task.done' }); +})); + +test('without grantMeta, prepareRunEnv emits what it always has', () => withRoot(async (root) => { + const events = []; + const env = await prepareRunEnv({ grant: { credentials: [] }, root, trace: { emit: (type, data) => events.push({ type, data }) } }); + await env.cleanup(); + assert.deepEqual(events, [{ type: 'credential.granted', data: { names: [] } }]); +})); + +const ask = async (answer) => { + const events = []; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approve: async () => answer, approveTimeoutMs: 1000, grantId: 'g-9', + }); + const result = await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + return { result, data: events[0].data }; +}; + +const BEN = { by: 'u-ben', role: 'credential-owner' }; + +test('an approver approves only with approved exactly true and a named owner', async () => { + for (const role of ['credential-owner', 'agent-owner']) { + assert.deepEqual((await ask({ approved: true, by: 'u-ben', role })).result, { outcome: 'selected', optionId: 'allow' }, role); + } + for (const approved of ['yes', 1, 'true', {}, [true], null, undefined]) { + const { result, data } = await ask({ approved, ...BEN }); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }, `approved ${JSON.stringify(approved)} must not approve`); + assert.equal(data.approval, 'failed'); + } +}); + +test('a bare true approves nothing: the ask fails and policy denies (CTO, 10-25)', async () => { + const { result, data } = await ask(true); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }); + assert.deepEqual([data.approval, data.decidedBy, data.decidedByRole], ['failed', 'policy', 'policy']); +}); + +test('a malformed answer approves nothing: the ask fails and policy denies; no decided line names nobody', async () => { + const malformed = [ + 'yes', 1, false, null, undefined, [true], {}, + { approved: true }, + { approved: true, by: 'u-ben' }, + { approved: true, role: 'credential-owner' }, + { approved: true, by: 'Ben Smith', role: 'credential-owner' }, + { approved: true, by: 'u-ben', role: 'boss' }, + { approved: true, by: 'u-ben', role: 'policy' }, + { approved: true, by: 'u-ben', role: null }, + { approved: false }, + ]; + for (const answer of malformed) { + const { result, data } = await ask(answer); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }, `${JSON.stringify(answer)} must not approve`); + assert.deepEqual([data.approval, data.decidedBy, data.decidedByRole], ['failed', 'policy', 'policy'], JSON.stringify(answer)); + } +}); + +test('the decision names who decided and as what, separate from the agent and its owner', async () => { + const { data } = await ask({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-1' }); + assert.equal(data.approval, 'approved'); + assert.equal(data.decidedBy, 'u-ben'); + assert.equal(data.decidedByRole, 'credential-owner'); + assert.equal(data.decisionCardId, 'card-1'); + assert.equal(data.grantId, 'g-9'); + const refused = await ask({ approved: false, by: 'u-ben', role: 'credential-owner' }); + assert.equal(refused.data.approval, 'rejected'); + assert.equal(refused.data.decidedBy, 'u-ben', 'a refusal is attributed too'); + // A malformed card id is dropped rather than trusted. + const odd = await ask({ approved: true, ...BEN, decisionCardId: '' }); + assert.equal(odd.data.decidedBy, 'u-ben'); + assert.equal('decisionCardId' in odd.data, false); +}); + +test('a pod member\'s answer decides nothing: the call stays parked until the owner answers (ADR-033 D5)', async () => { + const events = []; + const seen = []; + let ownerAnswers; + const ownerAnswer = new Promise((resolve) => { ownerAnswers = resolve; }); + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approveTimeoutMs: 5000, grantId: 'g-9', + askedOf: { id: 'u-ben', role: 'credential-owner' }, + approve: (input) => { + seen.push(input); + return seen.length === 1 ? { approved: true, by: 'u-cal', role: 'pod-member', decisionCardId: 'card-1' } : ownerAnswer; + }, + }); + const pending = decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + // Bounded: a policy that lets the pod member decide never asks again. + for (let i = 0; i < 200 && seen.length < 2; i += 1) await new Promise((resolve) => { setTimeout(resolve, 5); }); + assert.equal(seen.length, 2, 'the owner is asked again after the pod member\'s answer'); + assert.deepEqual(events.map((e) => e.type), ['permission.requested'], 'no permission.decided for the pod member\'s answer'); + assert.deepEqual(seen[1].refused, { by: 'u-cal', role: 'pod-member', reason: 'not_owner' }, 'the card is told why'); + assert.equal(seen[0].refused, undefined); + ownerAnswers({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-1' }); + assert.deepEqual(await pending, { outcome: 'selected', optionId: 'allow' }); + const decided = events.filter((e) => e.type === 'permission.decided'); + assert.equal(decided.length, 1); + assert.equal(decided[0].data.decidedBy, 'u-ben'); + assert.equal(decided[0].data.decidedByRole, 'credential-owner'); +}); + +test('an approver that only ever answers as a pod member fails the ask, and the policy default decides', async () => { + const events = []; + let calls = 0; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approveTimeoutMs: 5000, grantId: 'g-9', + approve: () => { calls += 1; return { approved: true, by: 'u-cal', role: 'pod-member' }; }, + }); + const result = await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }); + assert.equal(calls, MAX_REFUSED_ANSWERS + 1); + const [decided] = events.filter((e) => e.type === 'permission.decided'); + assert.deepEqual([decided.data.decidedBy, decided.data.decidedByRole], ['policy', 'policy']); +}); + +test('a valet policy with a trace says it records permissions itself; without one it does not', () => { + const recording = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { trace: { emit: () => {} } }); + assert.equal(recording.recordsPermissions, true); + assert.equal(Object.keys(recording).includes('recordsPermissions'), false, 'non-enumerable'); + assert.equal(createPermissionPolicy({ tools: { ask: ['fetch'] } }).recordsPermissions, undefined); +}); + +test('no answer in time is decided by policy', async () => { + const events = []; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approve: () => new Promise(() => {}), approveTimeoutMs: 10, + }); + await decide({ toolCall: { toolCallId: 'tc-1', title: 'x', kind: 'fetch' }, options: OPTIONS }); + assert.equal(events[0].data.approval, 'timed_out'); + assert.equal(events[0].data.decidedBy, 'policy'); + assert.equal(events[0].data.decidedByRole, 'policy'); +}); + +test('the question is redacted: a credential value in the ask never reaches disk', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0003', actor: { type: 'agent', id: 'u-agent-1', owner: 'u-ana' } }); + const grant = { credentials: [{ name: 'BROKER_TOKEN', from: 'resolver' }], tools: { ask: ['fetch'] } }; + const env = await prepareRunEnv({ grant, resolveSecret: async () => FROM_RESOLVER, root, trace: run }); + await env.cleanup(); + const decide = createPermissionPolicy(grant, { + trace: run, grantId: 'g-3', askedOf: { id: 'u-ben', role: 'credential-owner' }, approve: async () => ({ approved: true, ...BEN }), + }); + await decide({ toolCall: { toolCallId: 'tc-1', title: `Call the API with token ${FROM_RESOLVER}?`, kind: 'fetch' }, options: OPTIONS }); + const raw = readFileSync(join(root, 'trace', 'runs', 'run-va-0003.jsonl'), 'utf8'); + const asked = store.readRun('run-va-0003').find((e) => e.kind === 'permission.requested'); + assert.equal(asked.data.question, 'Call the API with token [redacted]?', 'positive control: the question was written, scrubbed'); + assert.ok(!raw.includes(FROM_RESOLVER)); + assert.equal(store.verifyRun('run-va-0003').ok, true); +})); + +const asks = async (askedOf, toolCall = { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, tools = { ask: ['fetch'] }) => { + const events = []; + const decide = createPermissionPolicy({ tools }, { + trace: { emit: (type, data) => events.push({ type, data }) }, grantId: 'g-5', askedOf, + approve: async (input) => { events.push({ type: 'approver-saw', data: input }); return { approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: input.decisionCardId }; }, + }); + await decide({ toolCall, options: OPTIONS }); + return events; +}; + +test('permission.requested is written first, asked of the host\'s choice for an ask and of the policy for a rule', async () => { + const events = await asks(() => ({ id: 'u-ben', role: 'credential-owner', decisionCardId: 'card-9' })); + assert.deepEqual(events.map((e) => e.type), ['permission.requested', 'approver-saw', 'permission.decided']); + assert.deepEqual(events[0].data, { + toolCallId: 'tc-1', options: OPTIONS.map(({ optionId, kind }) => ({ optionId, kind })), question: 'fetch it', + askedOf: { id: 'u-ben', role: 'credential-owner' }, grantId: 'g-5', decisionCardId: 'card-9', + }); + assert.equal(events[1].data.decisionCardId, 'card-9', 'the approver gets the card the ask rides on'); + assert.equal(events[2].data.decisionCardId, 'card-9'); + const byRule = await asks({ id: 'u-ben', role: 'credential-owner' }, { toolCallId: 'tc-2', title: 'cat file', kind: 'read' }, { allow: ['read'] }); + assert.deepEqual(byRule[0].data.askedOf, { id: 'grant:g-5', role: 'policy' }); + assert.equal(byRule.some((e) => e.type === 'approver-saw'), false); + for (const bad of [() => ({ id: 'Ben Smith', role: 'credential-owner' }), () => ({ id: 'u-ben' }), () => { throw new Error('x'); }]) { + assert.deepEqual((await asks(bad))[0].data.askedOf, { id: 'grant:g-5', role: 'policy' }, 'a malformed answer names nobody'); + } + assert.throws(() => createPermissionPolicy({ tools: {} }, { askedOf: { id: 'Ben Smith', role: 'policy' } }), /askedOf/); + const untitled = await asks({ id: 'u-ben', role: 'agent-owner' }, { toolCallId: 'tc-3', kind: 'fetch' }); + assert.equal(untitled[0].data.question, 'fetch call'); +}); + +test('the permission path writes the digest of the call\'s arguments, never the arguments', async () => { + const rawInput = { url: 'https://example.test/issues/88', body: { status: 'resolved' } }; + const events = await asks({ id: 'u-ben', role: 'credential-owner' }, { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch', rawInput }); + const digest = argsDigestOf(rawInput); + assert.match(digest, /^sha256:[0-9a-f]{64}$/); + assert.equal(events[0].data.argsDigest, digest); + assert.equal(events[2].data.argsDigest, digest); + assert.ok(!JSON.stringify([events[0].data, events[2].data]).includes('example.test')); + const none = await asks(undefined, { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }); + assert.equal('argsDigest' in none[1].data, false, 'no rawInput, no digest'); + assert.deepEqual(none.map((e) => e.type), ['approver-saw', 'permission.decided'], 'without askedOf, nothing new is emitted'); +}); diff --git a/packages/seats/test/supervisor-group-kill.test.js b/packages/seats/test/supervisor-group-kill.test.js new file mode 100644 index 0000000000..4de7ab9211 --- /dev/null +++ b/packages/seats/test/supervisor-group-kill.test.js @@ -0,0 +1,218 @@ +// A stalled turn's kill has to reach what the agent CLI spawned, not only the +// CLI. Each test here uses real processes: a shell that starts a grandchild +// (`sleep 300 &`), writes its pid, and waits on it, which is the shape of a CLI +// that started an MCP server. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { isRunning, readPid, reap, waitFor } from '../test-support/processes.js'; +import { + LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP, leadsOwnGroup, spawnInOwnGroup, trackedGroupCount, wantsOwnProcessGroup, +} from '../src/process-group.js'; + +const SRC = fileURLToPath(new URL('../src/', import.meta.url)); +const POSIX = process.platform !== 'win32'; +const STDIO = ['ignore', 'pipe', 'pipe']; + + +/** + * An adapter shaped like the real ones: it spawns through spawnInOwnGroup, + * asking for a group only when the supervisor offered one, and reports the + * child through onSpawn. It ignores the abort, so the supervisor's kill is the + * only thing that ends the turn. + */ +const grandchildAdapter = ({ pidFile, children, honourGroup = true }) => ({ + name: 'sh-grandchild', + spawn(prompt, ctx) { + return new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', `sleep 300 & echo $! > '${pidFile}'; wait`], { stdio: STDIO }, { + ownGroup: honourGroup && wantsOwnProcessGroup(ctx), + }); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + child.on('close', (code, signal) => { + if (ctx.signal.aborted) reject(ctx.signal.reason); + else reject(runFailure('exited', `sh exited ${code ?? signal}`)); + }); + }); + }, +}); + +const withHome = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-group-kill-')); + try { return await fn(home); } finally { rmSync(home, { recursive: true, force: true }); } +}; + +const STALL = { stallMs: 300, cancelGraceMs: 300, timeoutMs: 20000 }; + +test('the supervisor tells the adapter it may lead its own group', async () => withHome(async (home) => { + let seen; + const adapter = { name: 'spy', spawn: async (prompt, ctx) => { seen = ctx; return { text: 'ok' }; } }; + await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: join(home, 'trace') }) }); + assert.equal(seen[OWN_PROCESS_GROUP], true); +})); + +test('unasked, spawnInOwnGroup hands the options through untouched', () => { + const options = { cwd: '/x', env: { A: '1' }, stdio: STDIO }; + const calls = []; + const child = spawnInOwnGroup((...args) => { calls.push(args); return {}; }, 'cmd', ['a'], options); + assert.equal(calls[0][2], options); + assert.equal(Object.hasOwn(calls[0][2], 'detached'), false); + assert.equal(child[LEADS_PROCESS_GROUP], undefined); +}); + +test('a group-leading child is never inferred: detached without the marker is not a leader', { skip: !POSIX }, async () => { + const child = spawn('sleep', ['300'], { detached: true, stdio: 'ignore' }); + try { + await once(child, 'spawn'); + assert.equal(leadsOwnGroup(child), false); + child[LEADS_PROCESS_GROUP] = true; + assert.equal(leadsOwnGroup(child), true); + } finally { + child.kill('SIGKILL'); + await once(child, 'exit'); + } + assert.equal(leadsOwnGroup(child), false, 'a reaped child leads nothing, marked or not'); +}); + +test('a stalled turn\'s kill reaches the grandchild of a group-leading child', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + let grandchild; + try { + const run = superviseRun({ + adapter: grandchildAdapter({ pidFile, children }), prompt: 'p', ctx: {}, + store: createTraceStore({ root: join(home, 'trace') }), policy: STALL, + }); + grandchild = await readPid(pidFile); + assert.equal(leadsOwnGroup(children[0]), true); + assert.equal(isRunning(grandchild), true); + const out = await run; + assert.equal(out.outcome, 'stuck'); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} survived the kill`); + assert.ok(await waitFor(() => children[0].exitCode !== null || children[0].signalCode !== null), 'the leader survived'); + assert.ok(await waitFor(() => trackedGroupCount() === 0), 'the exited group is still tracked'); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +test('a child that does not lead its group is killed with child.kill, and no negative pid is signalled', { skip: !POSIX }, async (t) => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + const groupSignals = []; + const realKill = process.kill; + t.mock.method(process, 'kill', function kill(pid, signal) { + if (Number(pid) < 0) groupSignals.push([pid, signal]); + return realKill.call(process, pid, signal); + }); + let grandchild; + try { + const out = await (async () => { + const run = superviseRun({ + adapter: grandchildAdapter({ pidFile, children, honourGroup: false }), prompt: 'p', ctx: {}, + store: createTraceStore({ root: join(home, 'trace') }), policy: STALL, + }); + grandchild = await readPid(pidFile); + return run; + })(); + assert.equal(out.outcome, 'stuck'); + assert.equal(leadsOwnGroup(children[0]), false); + assert.ok(await waitFor(() => children[0].signalCode !== null), 'child.kill did not reach the child'); + assert.equal(children[0].signalCode, 'SIGKILL'); + assert.deepEqual(groupSignals, []); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +// ── The wrapper's own exit ────────────────────────────────────────────────── +// A group-leading child is outside the wrapper's group, so launchd's kill of +// the seat's group no longer reaches it. The wrapper itself has to kill it. + +const WRAPPER = (pidFile, mode) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { superviseRun } from ${JSON.stringify(pathToFileURL(join(SRC, 'supervisor.js')).href)}; +import { createTraceStore } from ${JSON.stringify(pathToFileURL(join(SRC, 'trace/store.js')).href)}; +import { observeSpawn } from ${JSON.stringify(pathToFileURL(join(SRC, 'runtime-observation.js')).href)}; +import { spawnInOwnGroup, wantsOwnProcessGroup } from ${JSON.stringify(pathToFileURL(join(SRC, 'process-group.js')).href)}; +const mode = ${JSON.stringify(mode)}; +if (mode === 'host-handler') { + // Like 'agent run': the host handles the signal and decides when to exit. + process.on('SIGINT', () => { process.stdout.write('host-handled\\n'); setTimeout(() => process.exit(7), 300); }); +} +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise(() => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', "sleep 300 & echo $! > '${pidFile}'; wait"], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + if (mode === 'exit') process.exit(3); + }, 20); + }); +}) }; +superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(`${pidFile}.trace`)} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +`; + +const runWrapper = async (home, mode, signal) => { + const pidFile = join(home, `grandchild-${mode}-${signal || 'none'}.pid`); + const script = join(home, `wrapper-${mode}-${signal || 'none'}.mjs`); + writeFileSync(script, WRAPPER(pidFile, mode)); + const wrapper = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + let out = ''; + wrapper.stdout.on('data', (chunk) => { out += chunk; }); + const exited = once(wrapper, 'exit'); + let leader; + let grandchild; + try { + assert.ok(await waitFor(() => /ready \d+/.test(out)), `wrapper never got ready: ${out}`); + leader = Number(out.match(/ready (\d+)/)[1]); + grandchild = await readPid(pidFile); + if (signal) { + assert.equal(isRunning(grandchild), true); + wrapper.kill(signal); + } + const [code, how] = await Promise.race([exited, new Promise((r) => { setTimeout(() => r([null, 'timeout']), 5000); })]); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} survived the wrapper (${mode}, ${signal})`); + assert.ok(await waitFor(() => !isRunning(leader)), `leader ${leader} survived the wrapper (${mode}, ${signal})`); + return { code, how, out }; + } finally { + if (grandchild) reap(grandchild); + if (leader) reap(leader); + wrapper.kill('SIGKILL'); + } +}; + +for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) { + test(`a wrapper killed by ${signal} takes its supervised groups with it, and still dies by ${signal}`, { skip: !POSIX }, async () => withHome(async (home) => { + const { code, how } = await runWrapper(home, 'plain', signal); + assert.equal(code, null); + assert.equal(how, signal, 'the wrapper no longer dies by the signal it was sent'); + })); +} + +test('a wrapper that exits takes its supervised groups with it', { skip: !POSIX }, async () => withHome(async (home) => { + const { code } = await runWrapper(home, 'exit', null); + assert.equal(code, 3); +})); + +test('a wrapper that handles the signal itself keeps its groups through the signal, and loses them when it exits', { skip: !POSIX }, async () => withHome(async (home) => { + const { code, out } = await runWrapper(home, 'host-handler', 'SIGINT'); + assert.match(out, /host-handled/); + assert.equal(code, 7); +})); diff --git a/packages/seats/test/supervisor-process.test.js b/packages/seats/test/supervisor-process.test.js new file mode 100644 index 0000000000..cb71038d67 --- /dev/null +++ b/packages/seats/test/supervisor-process.test.js @@ -0,0 +1,105 @@ +// Real processes against the fake provider: the hang is a genuine HTTP call +// that never answers, in a genuine child, so liveness has to fire on silence +// it did not stage, and the child has to actually die. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +// Says it is starting (a byte of progress), then waits on a completion the +// fake provider holds open forever. +const CHILD = ` +process.stdout.write('requesting\\n'); +fetch(process.argv[1] + '/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'fake-model', messages: [{ role: 'user', content: '[[stall]]' }] }), +}).then((r) => r.text()).then((t) => process.stdout.write(t)); +`; + +const nodeFetchAdapter = ({ honourAbort, children }) => ({ + name: 'node-fetch', + spawn(prompt, ctx) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['-e', CHILD, ctx.baseURL], { cwd: ctx.cwd, env: ctx.env, stdio: ['ignore', 'pipe', 'pipe'] }); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + let out = ''; + child.stdout.on('data', (chunk) => { out += chunk; }); + if (honourAbort) ctx.signal.addEventListener('abort', () => child.kill('SIGTERM'), { once: true }); + child.on('close', (code, signal) => { + ctx.trace.emit('process.exited', { code, signal }); + if (ctx.signal.aborted) reject(ctx.signal.reason); + else if (code === 0) resolve({ text: out }); + else reject(runFailure('exited', `node exited ${code ?? signal}`)); + }); + }); + }, +}); + +const isRunning = (pid) => { + try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } +}; + +const withProvider = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-sup-proc-')); + const fake = await startFakeProvider(); + try { + return await fn({ fake, home, store: createTraceStore({ root: join(home, 'trace') }) }); + } finally { + await fake.close(); + rmSync(home, { recursive: true, force: true }); + } +}; + +const settleChild = async (child) => { + if (child.exitCode === null && child.signalCode === null) { + let timer; + await Promise.race([once(child, 'exit'), new Promise((r) => { timer = setTimeout(r, 3000); })]); + clearTimeout(timer); + } +}; + +for (const honourAbort of [false, true]) { + test(`a real child hung on a stalled HTTP call is caught and killed (adapter ${honourAbort ? 'honours' : 'ignores'} abort)`, () => withProvider(async ({ fake, home, store }) => { + const children = []; + const ctx = { baseURL: fake.baseURL, cwd: home, env: { PATH: process.env.PATH, HOME: home } }; + const started = Date.now(); + const out = await superviseRun({ + adapter: nodeFetchAdapter({ honourAbort, children }), prompt: 'p', ctx, store, + policy: { stallMs: 400, cancelGraceMs: 400, timeoutMs: 10000 }, + }); + const elapsed = Date.now() - started; + + assert.equal(out.outcome, 'stuck'); + assert.equal(out.failure.kind, 'stuck'); + assert.ok(elapsed < 3000, `settled in ${elapsed}ms`); + assert.deepEqual(fake.requests.map((r) => r.directive), ['stall'], 'the child really made the call that hung'); + + assert.equal(children.length, 1); + const [child] = children; + await settleChild(child); + assert.ok(child.signalCode, `the child was killed by a signal (exit ${child.exitCode}, signal ${child.signalCode})`); + assert.equal(isRunning(child.pid), false, `pid ${child.pid} is gone`); + + const types = store.readRun(out.runId).map((e) => (e.type === 'liveness.recovery' ? `recovery:${e.data.action}` : e.type)); + if (honourAbort) { + assert.equal(child.signalCode, 'SIGTERM'); + assert.equal(out.failure.adapterIgnoredCancel, undefined); + assert.deepEqual(types, ['run.started', 'liveness.stalled', 'recovery:cancel', 'process.exited', 'run.settled']); + } else { + // Nobody but the supervisor stopped it: the kill came from the handle onSpawn gave it. + assert.equal(child.signalCode, 'SIGKILL'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.deepEqual(types, ['run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:kill', 'run.settled']); + } + })); +} diff --git a/packages/seats/test/supervisor-tree-and-deadline.test.js b/packages/seats/test/supervisor-tree-and-deadline.test.js new file mode 100644 index 0000000000..eb2aee56a5 --- /dev/null +++ b/packages/seats/test/supervisor-tree-and-deadline.test.js @@ -0,0 +1,225 @@ +// Two layers on top of the group kill, with real processes. +// +// The descendant walk: a CLI's child that moved to a group of its own (codex +// does this for every MCP server and tool command) is outside the group kill, +// so the kill also walks the process tree. A pid is signalled only if its ppid +// and start time still match the snapshot, so a reused pid is left alone. +// +// The SIGTERM deadline: a host that keeps the in-flight turn on SIGTERM keeps +// the groups too, and kills them at a deadline set before launchd's SIGKILL. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { + descendantsOf, killProcessTree, leadsOwnGroup, readProcess, readProcessTable, spawnInOwnGroup, wantsOwnProcessGroup, +} from '../src/process-group.js'; +import { isRunning, readPids, reap, waitFor } from '../test-support/processes.js'; + +const SRC = fileURLToPath(new URL('../src/', import.meta.url)); +const POSIX = process.platform !== 'win32'; + +const withHome = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-tree-')); + try { return await fn(home); } finally { rmSync(home, { recursive: true, force: true }); } +}; + +// A child that starts `count` grandchildren, each the leader of a new group +// (detached), writes their pids, and stays alive. +const NEW_GROUP_PARENT = ` +const { spawn } = require('node:child_process'); +const pids = []; +for (let i = 0; i < Number(process.argv[2]); i += 1) { + pids.push(spawn('sleep', ['300'], { detached: true, stdio: 'ignore' }).pid); +} +require('node:fs').writeFileSync(process.argv[1], pids.join(',')); +setInterval(() => {}, 1e6); +`; + +const startParent = (pidFile, count, ownGroup) => spawnInOwnGroup( + spawn, process.execPath, ['-e', NEW_GROUP_PARENT, pidFile, String(count)], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup }, +); + +test('the descendant table is read deepest first', () => { + const table = [ + { pid: 10, ppid: 1, started: 'a' }, { pid: 11, ppid: 10, started: 'b' }, { pid: 12, ppid: 11, started: 'c' }, + { pid: 13, ppid: 10, started: 'd' }, { pid: 20, ppid: 1, started: 'e' }, + ]; + assert.deepEqual(descendantsOf(table, 10).map((row) => [row.pid, row.depth]), [[12, 2], [11, 1], [13, 1]]); +}); + +test('a stall kill reaches a grandchild that started a group of its own', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + let grandchild; + try { + const adapter = { + name: 'new-group-parent', + spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = startParent(pidFile, 1, wantsOwnProcessGroup(ctx)); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + child.on('close', () => reject(ctx.signal.aborted ? ctx.signal.reason : runFailure('exited', 'parent exited'))); + }), + }; + const run = superviseRun({ + adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: join(home, 'trace') }), + policy: { stallMs: 300, cancelGraceMs: 300, timeoutMs: 20000 }, + }); + [grandchild] = await readPids(pidFile, 1); + assert.equal(leadsOwnGroup(children[0]), true); + assert.equal(isRunning(grandchild), true); + assert.equal((await run).outcome, 'stuck'); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} in its own group survived the kill`); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +test('a pid reused by another process is left alone; the descendant the snapshot named is killed', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchildren.pid'); + const parent = startParent(pidFile, 2, true); + // An unrelated process: not in the parent's tree at all. + const unrelated = spawn('sleep', ['300'], { stdio: 'ignore' }); + const unrelatedStarted = once(unrelated, 'spawn'); + let real; + let reused; + try { + [real, reused] = await readPids(pidFile, 2); + await unrelatedStarted; + const table = readProcessTable(); + assert.ok(table, 'ps is unavailable, so the walk cannot run'); + assert.ok(readProcess(reused), 'the reused stand-in is not running'); + // The snapshot as it would read if each pid had belonged to an earlier + // process: `reused` is the right parent's child but started at another + // time, and `unrelated` is named as the parent's child though it is not. + const stub = table.map((row) => (row.pid === reused ? { ...row, started: 'Thu Jan 1 00:00:00 1970' } : row)); + stub.push({ pid: unrelated.pid, ppid: parent.pid, started: readProcess(unrelated.pid).started }); + + killProcessTree(parent, 'SIGKILL', { table: () => stub }); + + assert.ok(await waitFor(() => !isRunning(real)), 'the walk did not kill the descendant the snapshot named'); + assert.ok(await waitFor(() => parent.signalCode !== null), 'the parent survived'); + await new Promise((r) => { setTimeout(r, 200); }); + assert.equal(isRunning(reused), true, 'a pid whose start time changed was signalled'); + assert.equal(isRunning(unrelated.pid), true, 'a pid whose parent changed was signalled'); + } finally { + for (const pid of [real, reused]) if (pid) reap(pid); + unrelated.kill('SIGKILL'); + parent.kill('SIGKILL'); + } +})); + +// ── SIGTERM in a host that keeps the turn ───────────────────────────────────── + +// Shaped like `agent run`: SIGTERM asks for a stop after the current turn and +// arms the deadline; when the turn is over, the host's exit path kills the +// groups. The turn ends when the test creates `go`; its child then says +// `done` and lingers, as a CLI shutting down can. +const HOST = ({ pidFile, goFile, deadlineMs }) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { superviseRun } from ${JSON.stringify(pathToFileURL(join(SRC, 'supervisor.js')).href)}; +import { createTraceStore } from ${JSON.stringify(pathToFileURL(join(SRC, 'trace/store.js')).href)}; +import { observeSpawn } from ${JSON.stringify(pathToFileURL(join(SRC, 'runtime-observation.js')).href)}; +import { + armSupervisedGroupDeadline, killSupervisedGroups, spawnInOwnGroup, wantsOwnProcessGroup, +} from ${JSON.stringify(pathToFileURL(join(SRC, 'process-group.js')).href)}; +let stopping = false; +process.on('SIGTERM', () => { + stopping = true; + process.stdout.write('stopping\\n'); + armSupervisedGroupDeadline(${deadlineMs}); +}); +const script = "sleep 300 & echo $! > '${pidFile}'; while [ ! -f '${goFile}' ]; do sleep 0.05; done; echo done; wait"; +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', script], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.stdout.on('data', (chunk) => { if (String(chunk).includes('done')) resolve({ text: 'done' }); }); + child.on('close', (code, signal) => reject(new Error('sh ended ' + (code ?? signal)))); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + }, 20); + }); +}) }; +const out = await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(`${pidFile}.trace`)} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +process.stdout.write('turn ' + out.outcome + '\\n'); +if (stopping) { killSupervisedGroups(); process.exit(0); } +`; + +const startHost = async (home, deadlineMs) => { + const pidFile = join(home, 'grandchild.pid'); + const goFile = join(home, 'go'); + const script = join(home, 'host.mjs'); + writeFileSync(script, HOST({ pidFile, goFile, deadlineMs })); + const host = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + const state = { host, goFile, out: '' }; + host.stdout.on('data', (chunk) => { state.out += chunk; }); + state.exited = once(host, 'exit'); + assert.ok(await waitFor(() => /ready \d+/.test(state.out)), `host never got ready: ${state.out}`); + state.leader = Number(state.out.match(/ready (\d+)/)[1]); + [state.grandchild] = await readPids(pidFile, 1); + return state; +}; + +const cleanup = (state) => { + if (!state) return; + reap(state.grandchild); + reap(state.leader); + state.host.kill('SIGKILL'); +}; + +test('SIGTERM mid-turn: the turn completes, then the groups die', { skip: !POSIX }, async () => withHome(async (home) => { + let state; + try { + state = await startHost(home, 10000); + state.host.kill('SIGTERM'); + assert.ok(await waitFor(() => /stopping/.test(state.out)), 'the host never saw SIGTERM'); + await new Promise((r) => { setTimeout(r, 400); }); + assert.equal(isRunning(state.grandchild), true, 'SIGTERM killed the turn\'s group'); + assert.equal(isRunning(state.leader), true, 'SIGTERM killed the turn\'s child'); + writeFileSync(state.goFile, ''); + const [code] = await Promise.race([state.exited, new Promise((r) => { setTimeout(() => r(['timeout']), 5000); })]); + assert.match(state.out, /turn ok/); + assert.equal(code, 0); + assert.ok(await waitFor(() => !isRunning(state.grandchild)), 'the group outlived the host'); + assert.ok(await waitFor(() => !isRunning(state.leader)), 'the leader outlived the host'); + } finally { + cleanup(state); + } +})); + +test('SIGTERM with the turn still running: the deadline kills the groups before ExitTimeOut', { skip: !POSIX }, async () => withHome(async (home) => { + // Stands in for launchd: ExitTimeOut 2 s, deadline 1 s (the CLI derives the + // real one from the seat's plist). + const exitTimeOutMs = 2000; + const deadlineMs = 1000; + let state; + try { + state = await startHost(home, deadlineMs); + const sent = Date.now(); + state.host.kill('SIGTERM'); + assert.ok(await waitFor(() => /stopping/.test(state.out)), 'the host never saw SIGTERM'); + await new Promise((r) => { setTimeout(r, deadlineMs / 2); }); + assert.equal(isRunning(state.grandchild), true, 'the group died before its deadline'); + assert.ok(await waitFor(() => !isRunning(state.grandchild), exitTimeOutMs), 'the group outlived ExitTimeOut'); + const diedAfter = Date.now() - sent; + assert.ok(diedAfter < exitTimeOutMs, `the group died ${diedAfter}ms after SIGTERM, past ExitTimeOut`); + assert.ok(await waitFor(() => !isRunning(state.leader), 1000), 'the leader outlived its group'); + } finally { + cleanup(state); + } +})); diff --git a/packages/seats/test/supervisor.test.js b/packages/seats/test/supervisor.test.js new file mode 100644 index 0000000000..7005ee8454 --- /dev/null +++ b/packages/seats/test/supervisor.test.js @@ -0,0 +1,431 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun, SUPERVISOR_DEFAULTS, DEFAULT_RETRY_ON } from '../src/supervisor.js'; +import * as seats from '../src/index.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { RunFailure, runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; + +// Small windows instead of a fake clock: every timer here is real, so the +// supervisor is exercised exactly as it runs, just in milliseconds. +const FAST = { stallMs: 60, cancelGraceMs: 60, timeoutMs: 0 }; +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const withStore = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-sup-')); + try { return await fn(createTraceStore({ root })); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const typesOf = (store, runId) => store.readRun(runId).map((e) => ( + e.type === 'liveness.recovery' ? `recovery:${e.data.action}` : e.type)); +const settledOf = (store, runId) => store.readRun(runId).filter((e) => e.type === 'run.settled'); + +/** Hangs until aborted, then rejects with the signal's reason, as the seam asks. */ +const hangsHonouringAbort = (calls = []) => ({ + name: 'hang-honour', + spawn(prompt, ctx) { + calls.push(ctx); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason), { once: true }); + }); + }, +}); + +/** A fake child: an EventEmitter with stdout/stderr and a kill that records. */ +const fakeChild = () => { + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.exitCode = null; + child.signalCode = null; + child.kills = []; + child.kill = (signal) => { + child.kills.push(signal); + child.signalCode = signal; + setImmediate(() => child.emit('exit', null, signal)); + return true; + }; + return child; +}; + +test('defaults are the documented windows and retry is off', () => { + assert.deepEqual({ ...SUPERVISOR_DEFAULTS }, { stallMs: 120000, cancelGraceMs: 10000, timeoutMs: 900000 }); + assert.deepEqual([...DEFAULT_RETRY_ON], ['stuck', 'exited', 'timeout']); + assert.equal(seats.superviseRun, superviseRun); +}); + +test('a successful run: started, the adapter\'s events, settled ok — in that order', () => withStore(async (store) => { + const adapter = { + name: 'ok', + async spawn(prompt, ctx) { + ctx.trace.emit('turn.prompt_sent', { chars: prompt.length }); + ctx.trace.emit('agent.message_chunk', { text: 'PONG' }); + return { text: 'PONG', newSessionId: 's-1' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'ping', store, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.deepEqual(out.result, { text: 'PONG', newSessionId: 's-1' }); + assert.equal(out.failure, undefined); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'turn.prompt_sent', 'agent.message_chunk', 'run.settled']); + const [settled] = settledOf(store, out.runId); + assert.equal(settled.data.outcome, 'ok'); + assert.equal(settled.data.attempts, 1); + assert.equal(typeof settled.data.durationMs, 'number'); + assert.equal(store.listRuns()[0].outcome, 'ok'); +})); + +test('the run\'s actor, task and pod reach the record, so run-owned entries are attributed', () => withStore(async (store) => { + const actor = { type: 'agent', id: 'u-agent-quire', owner: 'u-sam', onBehalfOf: { id: 'u-cy', role: 'assigner' } }; + const adapter = { + name: 'ok', + async spawn(prompt, ctx) { + ctx.trace.emit('agent.message_chunk', { text: 'PONG' }); + return { text: 'PONG' }; + }, + }; + const out = await superviseRun({ + adapter, prompt: 'ping', store, policy: FAST, taskId: 'TASK-548', podId: 'pod-sharpen', actor, + }); + assert.equal(out.outcome, 'ok'); + const entries = store.readRun(out.runId); + for (const entry of entries) { + assert.equal(entry.taskId, 'TASK-548', `${entry.type} carries the task`); + assert.equal(entry.podId, 'pod-sharpen', `${entry.type} carries the pod`); + } + const byType = Object.fromEntries(entries.map((e) => [e.type, e])); + assert.deepEqual(byType['run.started'].actor, actor, 'the supervisor\'s own run start is the run\'s'); + assert.deepEqual(byType['agent.message_chunk'].actor, actor, 'an emit that names no actor takes the run\'s'); + const [indexed] = store.listRuns(); + assert.equal(indexed.taskId, 'TASK-548'); + assert.equal(indexed.podId, 'pod-sharpen'); + assert.deepEqual(indexed.actor, actor); +})); + +test('a taskId without a podId is refused before the adapter is spawned', () => withStore(async (store) => { + let spawned = false; + const adapter = { name: 'ok', async spawn() { spawned = true; return { text: '' }; } }; + await assert.rejects( + superviseRun({ adapter, prompt: 'ping', store, policy: FAST, taskId: 'TASK-548' }), + /a taskId needs a podId/, + ); + assert.equal(spawned, false); +})); + +test('a hung adapter that honours abort settles stuck, with the reason on its signal', () => withStore(async (store) => { + const calls = []; + const stalled = []; + const out = await superviseRun({ + adapter: hangsHonouringAbort(calls), prompt: 'p', store, + policy: { ...FAST, cancelGraceMs: 2000, onStalled: (info) => stalled.push(info) }, + }); + assert.equal(out.outcome, 'stuck'); + assert.ok(out.failure instanceof RunFailure); + assert.equal(out.failure.kind, 'stuck'); + assert.equal(out.failure.adapterIgnoredCancel, undefined); + assert.equal(calls[0].signal.reason.kind, 'stuck', 'the adapter sees a RunFailure as the abort reason'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'liveness.stalled', 'recovery:cancel', 'run.settled']); + const events = store.readRun(out.runId); + const stall = events.find((e) => e.type === 'liveness.stalled').data; + assert.equal(stall.windowMs, 60); + assert.ok(stall.idleMs >= 60, `idleMs ${stall.idleMs} is at least the window`); + assert.deepEqual(events.find((e) => e.type === 'liveness.recovery').data, { action: 'cancel', attempt: 1 }); + assert.equal(settledOf(store, out.runId)[0].data.failure.kind, 'stuck'); + assert.equal(stalled.length, 1); + assert.equal(stalled[0].runId, out.runId); + assert.equal(stalled[0].attempt, 1); +})); + +test('a hung adapter that IGNORES abort still settles stuck, records kill, and kills its child', () => withStore(async (store) => { + const child = fakeChild(); + const adapter = { + name: 'hang-ignore', + spawn(prompt, ctx) { + ctx.onSpawn({ model: null, effort: null, at: 'x' }, child); + return new Promise(() => {}); // never settles, never looks at the signal + }, + }; + const started = Date.now(); + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + const elapsed = Date.now() - started; + assert.equal(out.outcome, 'stuck'); + assert.equal(out.failure.kind, 'stuck'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.match(out.failure.detail, /did not settle within 60ms of cancellation/); + assert.deepEqual(child.kills, ['SIGKILL']); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:kill', 'run.settled']); + assert.ok(elapsed < 1000, `settled in ${elapsed}ms, not left running`); +})); + +test('an ignored cancellation is not retried even when the kind is listed', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'hang-ignore', spawn() { calls += 1; return new Promise(() => {}); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 2 } } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(calls, 1, 'a second attempt could race the first, which may still be running'); +})); + +test('steady trace progress is never called stuck', () => withStore(async (store) => { + const adapter = { + name: 'steady', + async spawn(prompt, ctx) { + for (let i = 0; i < 15; i += 1) { // 300ms of work against a 60ms window + await delay(20); + ctx.trace.emit('agent.thought_chunk', { chars: 1 }); + } + return { text: 'done' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.ok(!typesOf(store, out.runId).some((t) => t.startsWith('liveness') || t.startsWith('recovery'))); +})); + +test('child stdout/stderr bytes count as progress, and the caller\'s onSpawn still receives every argument', () => withStore(async (store) => { + const seen = []; + const adapter = { + name: 'bytes', + async spawn(prompt, ctx) { + const child = fakeChild(); + observeSpawn(child, ctx.onSpawn, { model: 'm' }); + child.emit('spawn'); + for (let i = 0; i < 15; i += 1) { + await delay(20); + (i % 2 ? child.stdout : child.stderr).emit('data', Buffer.from('.')); + } + return { text: 'done' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { onSpawn: (...args) => seen.push(args) }, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.equal(seen.length, 1); + assert.equal(seen[0][0].model, 'm'); + assert.equal(typeof seen[0][1].kill, 'function', 'observeSpawn passes the child as the second argument'); + assert.ok(!typesOf(store, out.runId).includes('liveness.stalled')); +})); + +test('progress that stops part-way is caught, and idleMs is measured from the last signal', () => withStore(async (store) => { + const adapter = { + name: 'stops', + spawn(prompt, ctx) { + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)); + (async () => { for (let i = 0; i < 5; i += 1) { await delay(20); ctx.trace.emit('agent.thought_chunk', { chars: 1 }); } })(); + }); + }, + }; + const started = Date.now(); + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'stuck'); + const stall = store.readRun(out.runId).find((e) => e.type === 'liveness.stalled').data; + assert.ok(stall.idleMs >= 60 && stall.idleMs < 200, `idleMs ${stall.idleMs}`); + assert.ok(Date.now() - started >= 160, 'the window restarted on each signal'); +})); + +test('a refusal is never retried, even when listed in retry.on', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'refuses', async spawn() { calls += 1; throw runFailure('refusal', 'content policy'); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 3, on: ['refusal', 'stuck'] } } }); + assert.equal(calls, 1); + assert.equal(out.outcome, 'failed'); + assert.equal(out.attempts, 1); + assert.equal(out.failure.kind, 'refusal'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); + assert.deepEqual(settledOf(store, out.runId)[0].data.failure, { kind: 'refusal', detail: 'content policy' }); +})); + +test('stuck is retried once on the same runId, carrying the session, then succeeds', () => withStore(async (store) => { + const calls = []; + const adapter = { + name: 'flaky', + spawn(prompt, ctx) { + calls.push({ sessionId: ctx.sessionId }); + if (calls.length === 2) return Promise.resolve({ text: 'second time', newSessionId: 'sess-1' }); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => { + const failure = ctx.signal.reason; + failure.sessionId = 'sess-1'; // the adapter knew its session when it failed + reject(failure); + }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 1 } } }); + assert.equal(out.outcome, 'ok'); + assert.equal(out.attempts, 2); + assert.equal(out.result.text, 'second time'); + assert.deepEqual(calls, [{ sessionId: undefined }, { sessionId: 'sess-1' }]); + assert.deepEqual(typesOf(store, out.runId), [ + 'run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:retry', 'run.started', 'run.settled']); + const starts = store.readRun(out.runId).filter((e) => e.type === 'run.started').map((e) => e.data); + assert.deepEqual(starts, [{ adapter: 'flaky', attempt: 1 }, { adapter: 'flaky', sessionId: 'sess-1', attempt: 2 }]); + assert.equal(settledOf(store, out.runId)[0].data.attempts, 2); + assert.equal(store.listRuns().length, 1, 'a retry is the same run, not a new one'); +})); + +test('retry is off by default: a stuck run is attempted once', () => withStore(async (store) => { + const calls = []; + const out = await superviseRun({ adapter: hangsHonouringAbort(calls), prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000 } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(calls.length, 1); +})); + +test('retry stops at max and settles with the last failure', () => withStore(async (store) => { + const calls = []; + const out = await superviseRun({ adapter: hangsHonouringAbort(calls), prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000, retry: { max: 2 } } }); + assert.equal(calls.length, 3); + assert.equal(out.attempts, 3); + assert.equal(out.outcome, 'stuck'); + assert.equal(settledOf(store, out.runId).length, 1); +})); + +test('a kind left out of retry.on is not retried', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'exits', async spawn() { calls += 1; throw runFailure('exited', 'code 1'); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 2, on: ['stuck'] } } }); + assert.equal(calls, 1); + assert.equal(out.outcome, 'failed'); +})); + +test('the wall-clock timeout cancels a busy run with kind timeout, not stuck', () => withStore(async (store) => { + const adapter = { + name: 'busy', + spawn(prompt, ctx) { + return new Promise((_, reject) => { + const tick = setInterval(() => ctx.trace.emit('agent.thought_chunk', { chars: 1 }), 10); + ctx.signal.addEventListener('abort', () => { clearInterval(tick); reject(ctx.signal.reason); }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { stallMs: 1000, cancelGraceMs: 1000, timeoutMs: 120 } }); + assert.equal(out.outcome, 'failed'); + assert.equal(out.failure.kind, 'timeout'); + const types = typesOf(store, out.runId); + assert.ok(!types.includes('liveness.stalled')); + assert.deepEqual(types.filter((t) => !t.startsWith('agent.')), ['run.started', 'recovery:cancel', 'run.settled']); + assert.equal(settledOf(store, out.runId)[0].data.failure.kind, 'timeout'); +})); + +test('a timeout the adapter ignores still settles as timeout', () => withStore(async (store) => { + const adapter = { name: 'deaf', spawn: () => new Promise(() => {}) }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { stallMs: 0, cancelGraceMs: 40, timeoutMs: 60 } }); + assert.equal(out.failure.kind, 'timeout'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'recovery:cancel', 'recovery:kill', 'run.settled']); +})); + +test('a caller\'s signal cancels the run as cancelled, and a pre-aborted signal never spawns', () => withStore(async (store) => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 30); + const out = await superviseRun({ adapter: hangsHonouringAbort(), prompt: 'p', store, ctx: { signal: controller.signal }, policy: { ...FAST, stallMs: 1000, retry: { max: 3 } } }); + assert.equal(out.outcome, 'cancelled'); + assert.equal(out.failure.kind, 'cancelled'); + assert.equal(out.attempts, 1); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); + + let spawned = 0; + const out2 = await superviseRun({ adapter: { name: 'x', spawn: () => { spawned += 1; return {}; } }, prompt: 'p', store, ctx: { signal: AbortSignal.abort() }, policy: FAST }); + assert.equal(out2.outcome, 'cancelled'); + assert.equal(spawned, 0); +})); + +test('a bare Error settles failed with no kind, scrubbed, and does not throw', () => withStore(async (store) => { + const key = 'sk-or-v1-abcdefabcdefabcdef'; + const adapter = { name: 'bare', providerKeyEnv: 'OPENROUTER_API_KEY', spawn() { throw new Error(`exploded with ${key}`); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { env: { OPENROUTER_API_KEY: key } }, policy: FAST }); + assert.equal(out.outcome, 'failed'); + const { failure } = settledOf(store, out.runId)[0].data; + assert.equal(failure.kind, null); + assert.doesNotMatch(failure.detail, /abcdefabcdef/); + assert.match(failure.detail, /exploded with \[redacted\]/); +})); + +test('an adapter cannot write the supervisor\'s run.* events', () => withStore(async (store) => { + const adapter = { + name: 'forger', + async spawn(prompt, ctx) { + assert.equal(ctx.trace.emit('run.settled', { outcome: 'ok', durationMs: 0, attempts: 1 }), null); + assert.equal(ctx.trace.emit('run.started', { attempt: 9 }), null); + throw runFailure('exited', 'code 2'); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'failed'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); +})); + +test('exactly one run.settled on every path, and a late adapter answer writes nothing', () => withStore(async (store) => { + const lateHook = []; + const late = { + name: 'late', + spawn(prompt, ctx) { + // Ignores the abort, then answers after the supervisor gave up on it. + return new Promise((resolve) => setTimeout(() => { + lateHook.push(ctx.trace.emit('agent.message_chunk', { text: 'too late' })); + resolve({ text: 'too late' }); + }, 250)); + }, + }; + const paths = [ + { adapter: { name: 'ok', spawn: async () => ({ text: 'x' }) }, outcome: 'ok' }, + { adapter: hangsHonouringAbort(), outcome: 'stuck' }, + { adapter: { name: 'deaf', spawn: () => new Promise(() => {}) }, outcome: 'stuck' }, + { adapter: late, outcome: 'stuck' }, + { adapter: { name: 'refuse', spawn: async () => { throw runFailure('refusal', 'no'); } }, outcome: 'failed', policy: { retry: { max: 2, on: ['refusal'] } } }, + { adapter: hangsHonouringAbort(), outcome: 'stuck', policy: { retry: { max: 1 } } }, + { adapter: { name: 'deaf', spawn: () => new Promise(() => {}) }, outcome: 'failed', policy: { stallMs: 0, timeoutMs: 50 } }, + { adapter: { name: 'bare', spawn: () => { throw new Error('boom'); } }, outcome: 'failed' }, + ]; + const runIds = []; + for (const path of paths) { + const out = await superviseRun({ adapter: path.adapter, prompt: 'p', store, policy: { ...FAST, ...path.policy } }); + assert.equal(out.outcome, path.outcome, path.adapter.name); + runIds.push(out.runId); + } + await delay(300); // let the late adapter answer + assert.deepEqual(lateHook, [null], 'the late trace write was dropped'); + for (const runId of runIds) { + assert.equal(settledOf(store, runId).length, 1, runId); + assert.equal(store.readRun(runId).at(-1).type, 'run.settled', 'nothing follows the settle'); + } + assert.ok(store.listRuns({ limit: 100 }).every((row) => row.outcome !== 'running')); +})); + +test('an adapter that answers after the supervisor aborted it still settles with the abort reason', () => withStore(async (store) => { + const adapter = { + name: 'answers-anyway', + spawn(prompt, ctx) { + return new Promise((resolve) => { + ctx.signal.addEventListener('abort', () => resolve({ text: 'a reply nobody waited for' })); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000 } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(out.result, undefined); + assert.equal(settledOf(store, out.runId)[0].data.outcome, 'stuck'); +})); + +test('a cancelled attempt\'s late trace writes never reach the next attempt', () => withStore(async (store) => { + let calls = 0; + const adapter = { + name: 'stray', + spawn(prompt, ctx) { + calls += 1; + if (calls === 2) return delay(100).then(() => ({ text: 'ok' })); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => { + reject(ctx.signal.reason); + setTimeout(() => ctx.trace.emit('agent.message_chunk', { text: 'stale' }), 30); + }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, stallMs: 200, retry: { max: 1 } } }); + assert.equal(out.outcome, 'ok'); + assert.equal(out.attempts, 2); + assert.ok(!store.readRun(out.runId).some((e) => e.data?.text === 'stale')); +})); diff --git a/packages/seats/test/trace-store.test.js b/packages/seats/test/trace-store.test.js new file mode 100644 index 0000000000..8aaa196881 --- /dev/null +++ b/packages/seats/test/trace-store.test.js @@ -0,0 +1,77 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { EVENT_TYPES, traceOf, NULL_TRACE } from '../src/trace/events.js'; +import { RunFailure, runFailure, FAILURE_KINDS } from '../src/failures.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +const withRoot = (fn) => { const root = mkdtempSync(join(tmpdir(), 'seats-trace-')); try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); } }; + +test('a run appends events in seq order, settles once, and lists newest first', () => withRoot((root) => { + let t = 0; + const store = createTraceStore({ root, now: () => new Date(Date.UTC(2026, 9, 25, 0, 0, t++)) }); + const a = store.startRun({ runId: 'run-aaaaaa', adapter: 'stub' }); + a.emit('run.started', { adapter: 'stub', attempt: 1 }); + a.emit('agent.message_chunk', { text: 'PO' }); + a.emit('agent.message_chunk', { text: 'NG' }); + a.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + assert.equal(a.emit('agent.message_chunk', { text: 'late' }), null, 'nothing is written after settle'); + const b = store.startRun({ runId: 'run-bbbbbb', adapter: 'stub' }); + b.emit('run.started', { adapter: 'stub', attempt: 1 }); + + assert.deepEqual(store.readRun('run-aaaaaa').map((e) => [e.seq, e.type]), [ + [1, 'run.started'], [2, 'agent.message_chunk'], [3, 'agent.message_chunk'], [4, 'run.settled']]); + assert.deepEqual(store.listRuns().map((r) => [r.runId, r.outcome]), [['run-bbbbbb', 'running'], ['run-aaaaaa', 'ok']]); + assert.equal(statSync(join(root, 'runs', 'run-aaaaaa.jsonl')).mode & 0o777, 0o600); + assert.equal(statSync(join(root, 'runs')).mode & 0o777, 0o700); +})); + +test('unknown event types, duplicate and unsafe run ids are refused', () => withRoot((root) => { + const store = createTraceStore({ root }); + const run = store.startRun({ runId: 'run-cccccc' }); + assert.throws(() => run.emit('made.up', {}), /unknown trace event type/); + assert.throws(() => store.startRun({ runId: 'run-cccccc' }), /already exists/); + assert.throws(() => store.startRun({ runId: '../escape' }), /runId/); + assert.equal(store.readRun('../etc/passwd'), null); + assert.ok(EVENT_TYPES.includes('liveness.stalled')); +})); + +test('traceOf falls back to a no-op hook', () => { + assert.equal(traceOf({}), NULL_TRACE); + const hook = { emit() {} }; + assert.equal(traceOf({ trace: hook }), hook); +}); + +test('failures carry a kind, a retry flag, and scrub known credentials', () => { + const error = runFailure('refusal', 'provider said no to sk-or-v1-secretsecretsecret', { credentials: ['sk-or-v1-secretsecretsecret'] }); + assert.ok(error instanceof RunFailure); + assert.equal(error.kind, 'refusal'); + assert.equal(error.retryable, false); + assert.doesNotMatch(error.message, /secretsecret/); + assert.equal(FAILURE_KINDS.stuck.retryable, true); + assert.throws(() => new RunFailure('nope'), /unknown failure kind/); +}); + +test('fake provider answers stream and non-stream turns and honours directives', async () => { + const fake = await startFakeProvider(); + try { + const post = (body) => fetch(`${fake.baseURL}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + const plain = await (await post({ model: 'fake-model', messages: [{ role: 'user', content: 'hi' }] })).json(); + assert.equal(plain.choices[0].message.content, 'PONG'); + const custom = await (await post({ messages: [{ role: 'user', content: 'say [[reply:PONG-7]]' }] })).json(); + assert.equal(custom.choices[0].message.content, 'PONG-7'); + const streamed = await (await post({ stream: true, messages: [{ role: 'user', content: 'hi' }] })).text(); + assert.match(streamed, /"content":"PONG"/); + assert.match(streamed, /data: \[DONE\]/); + assert.equal((await post({ messages: [{ role: 'user', content: '[[refuse]]' }] })).status, 400); + const controller = new AbortController(); + const stalled = post({ messages: [{ role: 'user', content: '[[stall]]' }] }); + setTimeout(() => controller.abort(), 50); + const outcome = await Promise.race([stalled.then(() => 'answered'), new Promise((r) => setTimeout(() => r('held'), 150))]); + assert.equal(outcome, 'held'); + assert.deepEqual(fake.requests.map((r) => r.directive), ['reply', 'reply', 'reply', 'refuse', 'stall']); + } finally { await fake.close(); } +}); diff --git a/packages/seats/test/valet-env.test.js b/packages/seats/test/valet-env.test.js new file mode 100644 index 0000000000..6218f3a5a2 --- /dev/null +++ b/packages/seats/test/valet-env.test.js @@ -0,0 +1,181 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync as readdir, rmSync, statSync, writeFileSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { RunFailure } from '../src/failures.js'; + +const REAL_HOME = homedir(); +const GRANTED = 'sk-granted-0123456789abcdef'; +const WITHHELD = 'sk-withheld-fedcba9876543210'; + +const recorder = () => { const events = []; return { events, emit: (type, data) => events.push({ type, data }) }; }; +const withScratch = async (fn) => { + const dir = mkdtempSync(join(tmpdir(), 'seats-valet-')); + try { return await fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); } +}; + +// The child reports what it can see; the assertions run in the parent. +const PROBE = ` +const os = require('os'); const fs = require('fs'); const path = require('path'); +process.stdout.write(JSON.stringify({ + env: process.env, + homedir: os.homedir(), + claude: fs.existsSync(path.join(os.homedir(), '.claude')), + codex: fs.existsSync(path.join(os.homedir(), '.codex')), +}));`; +const probe = (env, cwd) => { + const child = spawnSync(process.execPath, ['-e', PROBE], { env, cwd, encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + return JSON.parse(child.stdout); +}; + +test('a real child sees only the granted key, a throwaway HOME, and no path to the real login', () => withScratch(async (root) => { + // A planted "real" home with both login dirs, so the result never depends on the host + // (a CI runner or a bare container has neither). + const loginHome = mkdtempSync(join(tmpdir(), 'seats-login-home-')); + for (const dir of ['.claude', '.codex']) { + mkdirSync(join(loginHome, dir), { mode: 0o700 }); + writeFileSync(join(loginHome, dir, 'credentials.json'), '{"planted":true}'); + } + // Positive control: a child given that HOME directly can reach both, so "not reachable" below is a real result. + const direct = probe({ ...process.env, HOME: loginHome }, loginHome); + assert.equal(direct.claude, true, 'positive control: the probe sees ~/.claude when HOME points at it'); + assert.equal(direct.codex, true, 'positive control: the probe sees ~/.codex when HOME points at it'); + // The parent's own env, with both secrets planted, is the base the run is cut from. + const baseEnv = { ...process.env, HOME: loginHome, OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD, CLAUDE_CONFIG_DIR: join(loginHome, '.claude') }; + const trace = recorder(); + const run = await prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }] }, baseEnv, root, trace, + }); + try { + const seen = probe(run.env, run.home); + assert.equal(seen.env.OPENAI_API_KEY, GRANTED, 'the granted key reaches the child'); + assert.equal(seen.env.ANTHROPIC_API_KEY, undefined, 'a parent secret the grant does not name is absent'); + assert.equal(seen.env.CLAUDE_CONFIG_DIR, undefined); + assert.ok(!JSON.stringify(seen.env).includes(WITHHELD)); + assert.equal(seen.env.HOME, run.home); + assert.equal(seen.homedir, run.home); + assert.ok(run.home.startsWith(root)); + for (const [name, value] of Object.entries(seen.env)) { + assert.ok(!value.includes(loginHome), `${name} leaks the login HOME path: ${value}`); + assert.ok(!value.includes(REAL_HOME), `${name} leaks the real HOME path: ${value}`); + } + assert.equal(seen.claude, false, '~/.claude is not reachable via HOME'); + assert.equal(seen.codex, false, '~/.codex is not reachable via HOME'); + for (const name of ['XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME', 'XDG_CACHE_HOME', 'TMPDIR']) { + assert.ok(seen.env[name].startsWith(`${run.home}/`), `${name} sits inside the run HOME`); + assert.equal(statSync(seen.env[name]).mode & 0o777, 0o700, `${name} is 0700`); + } + assert.equal(statSync(run.home).mode & 0o777, 0o700); + assert.deepEqual(run.granted, ['OPENAI_API_KEY']); + assert.deepEqual(trace.events, [{ type: 'credential.granted', data: { names: ['OPENAI_API_KEY'] } }]); + } finally { + await run.cleanup(); + rmSync(loginHome, { recursive: true, force: true }); + } + assert.equal(existsSync(run.home), false, 'cleanup removes the HOME'); +})); + +test('the base env keeps only the allowlist, and PATH loses entries under the real home', () => withScratch(async (root) => { + const baseEnv = { + HOME: REAL_HOME, PATH: `/usr/bin:${REAL_HOME}/.local/bin:/bin`, LANG: 'en_US.UTF-8', LC_ALL: 'C', TERM: 'xterm', TZ: 'UTC', + SSL_CERT_FILE: '/etc/ssl/cert.pem', NO_COLOR: '1', CI: '1', USER: 'someone', SSH_AUTH_SOCK: '/tmp/agent.sock', AWS_PROFILE: 'prod', GH_TOKEN: 'ghp_x', + }; + const run = await prepareRunEnv({ grant: {}, baseEnv, root }); + try { + assert.deepEqual(Object.keys(run.env).sort(), [ + 'CI', 'HOME', 'LANG', 'LC_ALL', 'NO_COLOR', 'PATH', 'SSL_CERT_FILE', 'TERM', 'TMPDIR', 'TZ', + 'XDG_CACHE_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME']); + assert.equal(run.env.PATH, '/usr/bin:/bin'); + } finally { await run.cleanup(); } +})); + +test('file and resolver credentials resolve; resolveSecret sees the name and spec', () => withScratch(async (root) => { + const keyFile = join(root, 'key.txt'); + writeFileSync(keyFile, 'from-file-value\n', { mode: 0o600 }); + const calls = []; + const run = await prepareRunEnv({ + grant: { credentials: [ + { name: 'FILE_KEY', from: 'file', ref: keyFile }, + { name: 'VAULT_KEY', from: 'resolver', ref: 'vault://team/key' }, + { name: 'RENAMED_KEY', from: 'env', ref: 'SOURCE_KEY' }, + ] }, + baseEnv: { SOURCE_KEY: 'from-env-value' }, + resolveSecret: async (name, spec) => { calls.push([name, spec.ref]); return 'from-resolver-value'; }, + root, + }); + try { + assert.equal(run.env.FILE_KEY, 'from-file-value'); + assert.equal(run.env.VAULT_KEY, 'from-resolver-value'); + assert.equal(run.env.RENAMED_KEY, 'from-env-value'); + assert.equal(run.env.SOURCE_KEY, undefined, 'the source name is not copied across too'); + assert.deepEqual(calls, [['VAULT_KEY', 'vault://team/key']]); + } finally { await run.cleanup(); } +})); + +test('an unresolvable credential fails credential_denied, emits its name only, and leaves no HOME', () => withScratch(async (root) => { + const cases = [ + { name: 'resolver returns nothing', resolveSecret: async () => undefined }, + { name: 'resolver throws with the value in its message', resolveSecret: async () => { throw new Error(`boom ${GRANTED}`); } }, + { name: 'resolver returns empty', resolveSecret: async () => '' }, + { name: 'no resolver at all', resolveSecret: undefined }, + ]; + for (const c of cases) { + const trace = recorder(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }, { name: 'VAULT_KEY', from: 'resolver' }] }, + baseEnv: { OPENAI_API_KEY: GRANTED }, resolveSecret: c.resolveSecret, root, trace, + }), (error) => { + assert.ok(error instanceof RunFailure, c.name); + assert.equal(error.kind, 'credential_denied'); + assert.match(error.message, /VAULT_KEY/); + assert.ok(!error.message.includes(GRANTED)); + return true; + }); + assert.deepEqual(trace.events, [{ type: 'credential.denied', data: { name: 'VAULT_KEY', reason: 'unresolved' } }], c.name); + } + await assert.rejects(prepareRunEnv({ grant: { credentials: [{ name: 'GONE', from: 'file', ref: join(root, 'missing') }] }, root }), /credential_denied/); + await assert.rejects(prepareRunEnv({ grant: { credentials: [{ name: 'GONE', from: 'env' }] }, baseEnv: {}, root }), /credential_denied/); + assert.deepEqual(readdir(root), [], 'every failed prepare removed its HOME'); +})); + +test('a credential the adapter requires but the grant does not name is denied before any HOME is made', () => withScratch(async (root) => { + const trace = recorder(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [] }, baseEnv: { ANTHROPIC_API_KEY: WITHHELD }, require: ['ANTHROPIC_API_KEY'], root, trace, + }), (error) => error.kind === 'credential_denied' && /not in the run grant/.test(error.message)); + assert.deepEqual(trace.events, [{ type: 'credential.denied', data: { name: 'ANTHROPIC_API_KEY', reason: 'not_granted' } }]); + assert.deepEqual(readdir(root), []); +})); + +test('no credential value ever reaches the serialized trace store', () => withScratch(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'valet-scan-1' }); + const keyFile = join(root, 'k'); + writeFileSync(keyFile, 'sk-file-secret-aaaaaaaaaaaa'); + const prepared = await prepareRunEnv({ + grant: { credentials: [ + { name: 'OPENAI_API_KEY', from: 'env' }, + { name: 'FILE_KEY', from: 'file', ref: keyFile }, + { name: 'VAULT_KEY', from: 'resolver' }, + ] }, + baseEnv: { OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD }, + resolveSecret: async () => 'sk-resolver-secret-bbbbbbbbbbbb', + root, + ctx: { trace: run }, + }); + await prepared.cleanup(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }, { name: 'VAULT_KEY', from: 'resolver' }] }, + baseEnv: { OPENAI_API_KEY: GRANTED }, resolveSecret: async () => { throw new Error(GRANTED); }, root, ctx: { trace: run }, + })); + const raw = readFileSync(join(root, 'trace', 'runs', 'valet-scan-1.jsonl'), 'utf8') + readFileSync(join(root, 'trace', 'index.jsonl'), 'utf8'); + assert.deepEqual(store.readRun('valet-scan-1').map((e) => e.type), ['credential.granted', 'credential.denied']); + for (const value of [GRANTED, WITHHELD, 'sk-file-secret-aaaaaaaaaaaa', 'sk-resolver-secret-bbbbbbbbbbbb']) { + assert.ok(!raw.includes(value), `trace carries a credential value: ${value}`); + } +})); diff --git a/packages/seats/test/valet-permissions.test.js b/packages/seats/test/valet-permissions.test.js new file mode 100644 index 0000000000..7f0611d2b4 --- /dev/null +++ b/packages/seats/test/valet-permissions.test.js @@ -0,0 +1,161 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { validateGrant, matchToolCall } from '../src/valet/grant.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import * as seats from '../src/index.js'; + +const OPTIONS = [ + { optionId: 'opt-always', name: 'Always allow', kind: 'allow_always' }, + { optionId: 'opt-once', name: 'Allow', kind: 'allow_once' }, + { optionId: 'opt-reject', name: 'Reject', kind: 'reject_once' }, + { optionId: 'opt-reject-always', name: 'Never', kind: 'reject_always' }, +]; +const request = (kind, title = 'a tool', options = OPTIONS) => ({ + sessionId: 's-1', toolCall: { toolCallId: `tc-${kind}`, title, kind }, options, +}); +const recorder = () => { const events = []; return { events, emit: (type, data) => events.push({ type, data }) }; }; + +test('precedence is deny > ask > allow > default deny', () => { + const grant = validateGrant({ tools: { + allow: ['read', 'search', 'execute:git *', '*:open *'], + ask: ['execute', 'edit:*.md'], + deny: ['execute:git push*', 'read:*.env*', 'edit:secrets/*'], + } }); + const table = [ + // [kind, title, decision, list, pattern] + ['read', 'README.md', 'allow', 'allow', 'read'], + ['read', 'config/.env.local', 'deny', 'deny', 'read:*.env*'], + ['search', 'grep TODO', 'allow', 'allow', 'search'], + ['execute', 'git status', 'ask', 'ask', 'execute'], + ['execute', 'git push origin main', 'deny', 'deny', 'execute:git push*'], + ['execute', 'npm test', 'ask', 'ask', 'execute'], + ['edit', 'notes.md', 'ask', 'ask', 'edit:*.md'], + ['edit', 'secrets/notes.md', 'deny', 'deny', 'edit:secrets/*'], + ['edit', 'src/a.js', 'deny', 'default', null], + ['fetch', 'open https://x', 'allow', 'allow', '*:open *'], + ['delete', 'rm -rf /', 'deny', 'default', null], + [undefined, 'mystery', 'deny', 'default', null], + ['not-a-kind', 'open thing', 'allow', 'allow', '*:open *'], + ]; + for (const [kind, title, decision, list, pattern] of table) { + assert.deepEqual(matchToolCall(grant, { kind, title }), { decision, rule: { list, pattern } }, `${kind} ${title}`); + } + // A missing kind means ACP's `other`, and `other` can be granted like any kind. + assert.equal(matchToolCall({ tools: { allow: ['other'] } }, { title: 'x' }).decision, 'allow'); + assert.equal(matchToolCall({}, { kind: 'read', title: 'x' }).decision, 'deny', 'an empty grant denies everything'); +}); + +test('globs are anchored and treat regex characters literally', () => { + const grant = { tools: { allow: ['execute:ls ?', 'read:a.b(c)'] } }; + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'ls a' }).decision, 'allow'); + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'ls ab' }).decision, 'deny'); + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'xls a' }).decision, 'deny'); + assert.equal(matchToolCall(grant, { kind: 'read', title: 'a.b(c)' }).decision, 'allow'); + assert.equal(matchToolCall(grant, { kind: 'read', title: 'aXb(c)' }).decision, 'deny'); +}); + +test('grant validation refuses unknown fields, bad names and bad patterns', () => { + const bad = [ + [null, /must be an object/], + [{ credential: [] }, /unknown field credential/], + [{ network: { allow: ['api.x'] } }, /network rules are not enforced yet/], + [{ tools: { allow: [], permit: [] } }, /unknown field tools\.permit/], + [{ tools: { allow: 'read' } }, /tools\.allow must be an array/], + [{ tools: { allow: ['write'] } }, /unknown tool kind "write"/], + [{ tools: { deny: ['execute:'] } }, /empty title glob/], + [{ tools: { ask: [''] } }, /non-empty string/], + [{ credentials: [{ name: 'openai_key', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: '1KEY', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: 'KEY-X', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: 'HOME', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'PATH', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'XDG_CONFIG_HOME', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'KEY', from: 'keychain' }] }, /from must be one of/], + [{ credentials: [{ name: 'KEY', from: 'env', value: 'sk-x' }] }, /unknown field credentials\[0\]\.value/], + [{ credentials: [{ name: 'KEY', from: 'file' }] }, /absolute path/], + [{ credentials: [{ name: 'KEY', from: 'file', ref: 'rel/key' }] }, /absolute path/], + [{ credentials: [{ name: 'KEY', from: 'env', ref: 'lower' }] }, /ENV_NAME for from: env/], + [{ credentials: [{ name: 'KEY', from: 'env' }, { name: 'KEY', from: 'resolver' }] }, /granted twice/], + ]; + for (const [grant, message] of bad) assert.throws(() => validateGrant(grant), message, JSON.stringify(grant)); + const ok = validateGrant({ credentials: [{ name: 'OPENAI_API_KEY', from: 'resolver', ref: 'vault://k' }], tools: { allow: ['read'] } }); + assert.ok(Object.isFrozen(ok) && Object.isFrozen(ok.credentials[0])); + assert.equal(validateGrant(ok), ok, 'a validated grant passes through'); +}); + +test('allow selects allow_once and never allow_always', async () => { + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { allow: ['read'] } }, { trace }); + assert.deepEqual(await decide(request('read')), { outcome: 'selected', optionId: 'opt-once' }); + // Only allow_always offered: an allow rule still must not pick it. + const onlyAlways = [OPTIONS[0], OPTIONS[2]]; + assert.deepEqual(await decide(request('read', 'x', onlyAlways)), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(await decide(request('read', 'x', [OPTIONS[0]])), { outcome: 'cancelled' }); + assert.deepEqual(trace.events[0], { type: 'permission.decided', data: { + outcome: 'selected', optionId: 'opt-once', kind: 'allow_once', toolCallId: 'tc-read', rule: { list: 'allow', pattern: 'read' }, + } }); +}); + +test('deny selects reject_once, or cancelled when there is no reject_once option', async () => { + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { deny: ['execute'] } }, { trace }); + assert.deepEqual(await decide(request('execute')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(await decide(request('execute', 'x', [OPTIONS[0], OPTIONS[1], OPTIONS[3]])), { outcome: 'cancelled' }); + assert.deepEqual(await decide(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }, 'default deny'); + assert.deepEqual(await decide({}), { outcome: 'cancelled' }, 'a malformed request is denied, not thrown'); + assert.deepEqual(trace.events.map((e) => [e.data.outcome, e.data.rule.list]), [ + ['selected', 'deny'], ['cancelled', 'deny'], ['selected', 'default'], ['cancelled', 'default']]); +}); + +const BEN = { by: 'u-ben', role: 'credential-owner' }; + +test('ask: approved allows once, rejected denies, and approve sees the tool call and rule', async () => { + const seen = []; + const trace = recorder(); + let verdict = { approved: true, ...BEN }; + const decide = createPermissionPolicy({ tools: { ask: ['execute'] } }, { + trace, approve: async (input) => { seen.push(input); return verdict; }, + }); + const req = request('execute', 'npm test'); + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-once' }); + verdict = { approved: false, ...BEN }; + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }); + verdict = { approved: 'yes', ...BEN }; + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }, 'only literal true approves'); + assert.deepEqual(seen[0], { toolCall: req.toolCall, grantRule: { list: 'ask', pattern: 'execute' } }); + assert.deepEqual(trace.events.map((e) => [e.data.kind, e.data.approval, e.data.decidedBy]), [ + ['allow_once', 'approved', 'u-ben'], ['reject_once', 'rejected', 'u-ben'], ['reject_once', 'failed', 'policy']]); +}); + +test('ask: no answer in time denies; a throwing or missing approver denies', async () => { + const trace = recorder(); + const started = Date.now(); + const slow = createPermissionPolicy({ tools: { ask: ['edit'] } }, { + trace, approveTimeoutMs: 30, approve: () => new Promise(() => {}), + }); + assert.deepEqual(await slow(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.ok(Date.now() - started < 1000); + const throwing = createPermissionPolicy({ tools: { ask: ['edit'] } }, { trace, approve: () => { throw new Error('card service down'); } }); + assert.deepEqual(await throwing(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + const none = createPermissionPolicy({ tools: { ask: ['edit'] } }, { trace }); + assert.deepEqual(await none(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(trace.events.map((e) => e.data.approval), ['timed_out', 'failed', 'failed']); +}); + +test('ask: a cancelled run answers a pending ask with cancelled', async () => { + const controller = new AbortController(); + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace, signal: controller.signal, approveTimeoutMs: 10_000, approve: () => new Promise(() => {}), + }); + const pending = decide(request('fetch')); + setTimeout(() => controller.abort(), 5); + assert.deepEqual(await pending, { outcome: 'cancelled' }); + assert.equal(trace.events[0].data.approval, 'aborted'); +}); + +test('the valet API is exported from the package entry', () => { + for (const name of ['validateGrant', 'matchToolCall', 'prepareRunEnv', 'createPermissionPolicy', 'TOOL_KINDS', 'BASE_ENV_ALLOWLIST']) { + assert.ok(name in seats, name); + } +}); diff --git a/packages/seats/test/vectors/args-digest.json b/packages/seats/test/vectors/args-digest.json new file mode 100644 index 0000000000..6640506027 --- /dev/null +++ b/packages/seats/test/vectors/args-digest.json @@ -0,0 +1,359 @@ +{ + "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested (a non-finite number, undefined arguments, or a lone surrogate in any string or key), and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape. A writer outside JavaScript must parse every JSON number as an IEEE 754 double before serializing it (as JSON.parse does), then print its shortest round-trip form as ECMAScript does: 9007199254740993 in the input is the double 9007199254740992, and is written that way. Strings are UTF-8 with U+2028 and U+2029 raw, not escaped.", + "vectors": [ + { + "name": "flat object, keys reordered", + "args": { + "b": 2, + "a": 1 + }, + "canonical": "{\"a\":1,\"b\":2}", + "digest": "sha256:43258cff783fe7036d8a43033f830adfc60ec037382473548ac742b888292777" + }, + { + "name": "nested objects and arrays sort keys at every depth", + "args": { + "z": { + "y": [ + { + "d": 1, + "c": 2 + } + ], + "x": null + }, + "a": { + "c": true, + "b": false + } + }, + "canonical": "{\"a\":{\"b\":false,\"c\":true},\"z\":{\"x\":null,\"y\":[{\"c\":2,\"d\":1}]}}", + "digest": "sha256:e361ba9cd9b9c2b05358b3a26828db49e7e3799a8ab9bdb2aa257247df0b2aab" + }, + { + "name": "keys sort by UTF-16 code units, not code points or UTF-8 bytes", + "args": { + "Ａ": 1, + "😀": 2, + "é": 3, + "z": 4, + "Z": 5 + }, + "canonical": "{\"Z\":5,\"z\":4,\"é\":3,\"😀\":2,\"Ａ\":1}", + "digest": "sha256:8777efd1f0f6d62748dd03f53d88c1228d506e1efd54aafc6e347ee191d15bc5" + }, + { + "name": "empty object", + "args": {}, + "canonical": "{}", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + { + "name": "empty array", + "args": [], + "canonical": "[]", + "digest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "name": "top-level array keeps order", + "args": [ + 3, + 1, + 2 + ], + "canonical": "[3,1,2]", + "digest": "sha256:51bda7ab4e44726cde71fcb6e4b515357059bb6b6dd5146d1fc50f73f11678c6" + }, + { + "name": "deeply nested arrays", + "args": [ + 1, + [ + 2, + [ + 3, + [] + ] + ] + ], + "canonical": "[1,[2,[3,[]]]]", + "digest": "sha256:2ae49150ac570d8d71784d5c84619d21a59e217faf3c632b0ef7a5477b61eb4d" + }, + { + "name": "top-level string", + "args": "git push origin main", + "canonical": "\"git push origin main\"", + "digest": "sha256:0703afb7169909864706a34e00de8cb86207aa3857e72539f34fe6cfeccd1e91" + }, + { + "name": "top-level null", + "args": null, + "canonical": "null", + "digest": "sha256:74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b" + }, + { + "name": "booleans and null", + "args": { + "t": true, + "f": false, + "n": null + }, + "canonical": "{\"f\":false,\"n\":null,\"t\":true}", + "digest": "sha256:22e00dc2f7b01420f940fbdbfbdf34fa0667cc6500186495023ba37722cbd05e" + }, + { + "name": "non-ASCII strings are written raw, not escaped", + "args": { + "s": "héllo wörld 日本 😀" + }, + "canonical": "{\"s\":\"héllo wörld 日本 😀\"}", + "digest": "sha256:44832c661e31ea40b67dd594e8b047c2f478fd93007426874dca007289058847" + }, + { + "name": "control characters use short escapes, then \\u00XX lowercase", + "args": { + "s": "\b\f\n\r\t\u0000\u0007\u001f" + }, + "canonical": "{\"s\":\"\\b\\f\\n\\r\\t\\u0000\\u0007\\u001f\"}", + "digest": "sha256:668550beedb0365aa4517dfe85faa784af491a2b29e94205e9b4e2da80298ac7" + }, + { + "name": "quote and backslash are escaped; / < > & DEL U+2028 are not", + "args": { + "s": "\"\\/<>&

" + }, + "canonical": "{\"s\":\"\\\"\\\\/<>&

\"}", + "digest": "sha256:2cd92a4cd6d75d7908756eddfd66f0b4bd361808bc32ea278fcb7dbde1f0eefa" + }, + { + "name": "U+2028 and U+2029 are written raw, never escaped", + "args": { + "s": "a
b
c" + }, + "canonical": "{\"s\":\"a
b
c\"}", + "digest": "sha256:5b2dc062ac2d05c4b51a388dd0e99e9b161c6c6f0fefeef39629890a42902b84" + }, + { + "name": "a lone high surrogate in a string means no digest", + "args": { + "s": "\ud800" + }, + "error": true + }, + { + "name": "a lone low surrogate nested in an array means no digest", + "args": { + "a": [ + "ok", + [ + "x\udc00y" + ] + ] + }, + "error": true + }, + { + "name": "a surrogate pair in the wrong order means no digest", + "args": { + "s": "\ude00\ud83d" + }, + "error": true + }, + { + "name": "a lone surrogate in a key means no digest", + "args": { + "k\ud800": 1 + }, + "error": true + }, + { + "name": "1.0 is written as 1", + "args": { + "n": { + "$number": "1.0" + } + }, + "canonical": "{\"n\":1}", + "digest": "sha256:2bfd14f43d17fc7cea24e0917a8879b4b2f880b8baeec1b9d90fbaad655e71bd" + }, + { + "name": "-0 is written as 0", + "args": { + "n": { + "$number": "-0" + } + }, + "canonical": "{\"n\":0}", + "digest": "sha256:f3013f933b9fb80ab6d995e7ad9da36f683837ba1d81e950c943d40111eac2f0" + }, + { + "name": "1e20 is written in full", + "args": { + "n": { + "$number": "1e20" + } + }, + "canonical": "{\"n\":100000000000000000000}", + "digest": "sha256:58d2d5b8dd4228ab5775ce84f996718fa19ed49872271e9649b70cebaca26a09" + }, + { + "name": "1e21 switches to exponent form with a plus sign", + "args": { + "n": { + "$number": "1e21" + } + }, + "canonical": "{\"n\":1e+21}", + "digest": "sha256:f1ee2b60ee95a3170fdc07a577e5f3514ced26867443d69da265acadead81007" + }, + { + "name": "0.000001 stays decimal", + "args": { + "n": { + "$number": "0.000001" + } + }, + "canonical": "{\"n\":0.000001}", + "digest": "sha256:28343867a0be00aee19f81aa90cfd6c646878b9303fa15410d16bfd3f8894578" + }, + { + "name": "1e-7 switches to exponent form", + "args": { + "n": { + "$number": "1e-7" + } + }, + "canonical": "{\"n\":1e-7}", + "digest": "sha256:747d6d23b64d1b2d579adb832b44de31c91c875bbef7a8e397f5d183a746b54b" + }, + { + "name": "0.1 + 0.2 keeps the shortest round-trip digits", + "args": { + "n": { + "$number": "0.30000000000000004" + } + }, + "canonical": "{\"n\":0.30000000000000004}", + "digest": "sha256:1d8a8e16d9ee625f1fb5ef505413efa212cc020b272ed0f9b3d3d81acf593fde" + }, + { + "name": "small and large finite extremes", + "args": { + "min": { + "$number": "5e-324" + }, + "max": { + "$number": "1.7976931348623157e308" + } + }, + "canonical": "{\"max\":1.7976931348623157e+308,\"min\":5e-324}", + "digest": "sha256:a49cfc2a701d0651c438486e57d8c53c451fb88c58de9be9be055a6e5f5dd13d" + }, + { + "name": "negative and fractional numbers", + "args": { + "a": { + "$number": "-1.5" + }, + "b": { + "$number": "-123" + }, + "c": { + "$number": "3.14159" + } + }, + "canonical": "{\"a\":-1.5,\"b\":-123,\"c\":3.14159}", + "digest": "sha256:fd4545080eaf9490b1e10de2680fc77a1a90886c9c494a05f1144cdb33d38ae0" + }, + { + "name": "an integer past 2^53: 9007199254740993 on the wire is the double 9007199254740992", + "args": { + "n": 9007199254740992 + }, + "canonical": "{\"n\":9007199254740992}", + "digest": "sha256:66c87d9cb3014e05a11baa97df62282d89d425f22ee15816577c84534e2ef1bb" + }, + { + "name": "large integers print their shortest round-trip digits, then zeros: 2^60 is 1152921504606847000", + "args": [ + 9007199254740994, + 1152921504606847000, + 10000000000000000 + ], + "canonical": "[9007199254740994,1152921504606847000,10000000000000000]", + "digest": "sha256:0e2b45d1064af2a28028209676fbd29080eca67ae85a2ad6582eb0efec521429" + }, + { + "name": "undefined in an object drops the key", + "args": { + "a": 1, + "b": { + "$undefined": true + } + }, + "canonical": "{\"a\":1}", + "digest": "sha256:015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862" + }, + { + "name": "undefined nested in an object drops only that key", + "args": { + "a": { + "b": { + "$undefined": true + }, + "c": 2 + } + }, + "canonical": "{\"a\":{\"c\":2}}", + "digest": "sha256:793fe66a2d4239e3e64774ce7ee90c6e1931ff33712ff05a17956a52e577c753" + }, + { + "name": "undefined in an array becomes null", + "args": [ + 1, + { + "$undefined": true + }, + 3 + ], + "canonical": "[1,null,3]", + "digest": "sha256:7db5b36a1553a4e31914df4beb60752bd07320e3056ac1d3790660c4cad6a173" + }, + { + "name": "NaN anywhere means no digest", + "args": { + "a": 1, + "b": { + "$number": "NaN" + } + }, + "error": true + }, + { + "name": "Infinity nested in an array means no digest", + "args": { + "a": [ + 1, + { + "$number": "Infinity" + } + ] + }, + "error": true + }, + { + "name": "-Infinity means no digest", + "args": { + "$number": "-Infinity" + }, + "error": true + }, + { + "name": "undefined arguments mean no digest", + "args": { + "$undefined": true + }, + "error": true + } + ] +}