From 9d3e5ec20e73f18fb6e4f098ce33fdd5a4b46c34 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:12:08 -0700 Subject: [PATCH 01/20] refactor(cli): move the adapter contract, stub and Hermes into packages/seats D62 extract-in-place, PR 1. No behaviour change. packages/seats (private, Apache-2.0) now holds the adapter contract and registry, the upstream-refusal naming, spawn observation, the seat-model check and two adapters. Hermes becomes a factory that takes Commonly's policy as a host object, so the package imports nothing from Commonly. The old cli paths re-export or bind it, so no caller and no existing test changes. The CLI ships the package as a bundled file: dependency. Two publish fixes come with it: - npm pack on an uninstalled tree exits 0 and silently drops the bundle. cli's prepack now refuses that case. - npm-publish installs before its compare step and triggers on packages/seats. seats-package.test.mjs covers: - the import boundary (mutation-checked); - that the tarball carries every package file (mutation-checked); - that the registry serves the bound adapters; - the neutral defaults. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- .github/workflows/npm-publish.yml | 13 +- cli/__tests__/seats-package.test.mjs | 80 +++++ cli/package-lock.json | 16 + cli/package.json | 7 +- cli/scripts/require-bundled-seats.mjs | 11 + cli/src/lib/adapters/hermes.js | 207 +------------ cli/src/lib/adapters/seat-model.js | 17 +- cli/src/lib/adapters/stub.js | 25 +- cli/src/lib/runtime-observation.js | 30 +- cli/src/lib/upstream-refusal.js | 355 +--------------------- packages/seats/README.md | 9 + packages/seats/package.json | 19 ++ packages/seats/src/adapters/hermes.js | 206 +++++++++++++ packages/seats/src/adapters/stub.js | 24 ++ packages/seats/src/contract.js | 48 +++ packages/seats/src/index.js | 6 + packages/seats/src/runtime-observation.js | 27 ++ packages/seats/src/seat-model.js | 17 ++ packages/seats/src/upstream-refusal.js | 354 +++++++++++++++++++++ 19 files changed, 855 insertions(+), 616 deletions(-) create mode 100644 cli/__tests__/seats-package.test.mjs create mode 100644 cli/scripts/require-bundled-seats.mjs create mode 100644 packages/seats/README.md create mode 100644 packages/seats/package.json create mode 100644 packages/seats/src/adapters/hermes.js create mode 100644 packages/seats/src/adapters/stub.js create mode 100644 packages/seats/src/contract.js create mode 100644 packages/seats/src/index.js create mode 100644 packages/seats/src/runtime-observation.js create mode 100644 packages/seats/src/seat-model.js create mode 100644 packages/seats/src/upstream-refusal.js diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index a547e07ba1..78e1641fb1 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -49,6 +49,7 @@ on: branches: [ main ] paths: - 'cli/**' + - 'packages/seats/**' # bundled into the cli tarball (bundleDependencies) - 'commonly-mcp/**' - 'docs/agents/skills/commonly/SKILL.md' # cli prepublishOnly copies it in workflow_dispatch: @@ -112,6 +113,14 @@ jobs: with: node-version: '22' + # Before the compare, not just before publish: the cli bundles + # packages/seats, and `npm pack` on a tree with no node_modules exits 0 + # with the bundle silently missing. The compare would then see every + # tree as different from what npm serves. cli's prepack now refuses + # that case, so a missing install fails here, by name. + - name: Install + run: npm ci --ignore-scripts + - name: Decide whether this tree holds anything unpublished id: cmp shell: bash @@ -237,10 +246,6 @@ jobs: done true - - name: Install - if: steps.cmp.outputs.action == 'publish' - run: npm ci --ignore-scripts - - name: Publish if: steps.cmp.outputs.action == 'publish' # No NODE_AUTH_TOKEN: npm exchanges this job's OIDC id-token with the diff --git a/cli/__tests__/seats-package.test.mjs b/cli/__tests__/seats-package.test.mjs new file mode 100644 index 0000000000..ef8bb984f8 --- /dev/null +++ b/cli/__tests__/seats-package.test.mjs @@ -0,0 +1,80 @@ +/** + * packages/seats is the runtime the CLI bundles (D62, extract-in-place). These + * tests hold the two properties the move depends on: the package imports + * nothing from Commonly, and the CLI tarball carries it. + */ +import { execFileSync } from 'child_process'; +import { readdirSync, readFileSync, statSync } from 'fs'; +import { builtinModules } from 'module'; +import { dirname, join, relative, resolve, sep } from 'path'; +import { fileURLToPath } from 'url'; +import { createHermesAdapter, createRegistry, stub } from '@commonlyai/seats'; +import cliHermes from '../src/lib/adapters/hermes.js'; +import { getAdapter } from '../src/lib/adapters/index.js'; + +const CLI = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const SEATS = resolve(CLI, '..', 'packages', 'seats'); + +const sourceFiles = (dir) => readdirSync(dir).flatMap((name) => { + const full = join(dir, name); + return statSync(full).isDirectory() ? sourceFiles(full) : (full.endsWith('.js') ? [full] : []); +}); + +const specifiers = (code) => [ + ...code.matchAll(/(?:^|\n)\s*(?:import|export)\b[^'"`;]*?from\s*['"]([^'"]+)['"]/g), + ...code.matchAll(/(?:^|\n)\s*import\s*['"]([^'"]+)['"]/g), + ...code.matchAll(/\bimport\(\s*['"]([^'"]+)['"]\s*\)/g), +].map((match) => match[1]); + +describe('packages/seats', () => { + test('imports only node built-ins and its own modules', () => { + const builtins = new Set(builtinModules); + const files = sourceFiles(join(SEATS, 'src')); + expect(files.length).toBeGreaterThan(5); + const outside = []; + for (const file of files) { + for (const spec of specifiers(readFileSync(file, 'utf8'))) { + if (spec.startsWith('node:') || builtins.has(spec)) continue; + const target = spec.startsWith('.') ? resolve(dirname(file), spec) : null; + if (target && (target + sep).startsWith(join(SEATS, 'src') + sep)) continue; + outside.push(`${relative(SEATS, file)} → ${spec}`); + } + } + expect(outside).toEqual([]); + }); + + test('the boundary scan sees a bad import', () => { + expect(specifiers("import x from '../../cli/src/lib/environment.js';\nexport { y } from 'commander';")) + .toEqual(['../../cli/src/lib/environment.js', 'commander']); + }); + + test('the CLI registry serves the bound adapters', () => { + expect(getAdapter('stub')).toBe(stub); + expect(getAdapter('hermes')).toBe(cliHermes); + expect(Object.keys(cliHermes).sort()).toEqual( + Object.keys(createHermesAdapter()).sort(), + ); + }); + + test('a host-less Hermes adapter takes the neutral defaults', async () => { + const hermes = createHermesAdapter(); + expect(hermes.requiresModel).toBe(false); + await expect(hermes.validateEnvironment({ sandbox: { trust: 'public' } })) + .rejects.toThrow('no enforced confinement adapter exists'); + }); + + test('a registry refuses duplicate names', () => { + expect(() => createRegistry([stub, stub])).toThrow('duplicate adapter name: stub'); + expect(createRegistry([stub]).get('stub')).toBe(stub); + }); + + test('the CLI tarball bundles every package source file', () => { + const [pack] = JSON.parse(execFileSync('npm', ['pack', '--dry-run', '--json', '--ignore-scripts'], { + cwd: CLI, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], + })); + const packed = new Set(pack.files.map((file) => file.path)); + const expected = sourceFiles(join(SEATS, 'src')) + .map((file) => `node_modules/@commonlyai/seats/${relative(SEATS, file).split(sep).join('/')}`); + expect(expected.filter((path) => !packed.has(path))).toEqual([]); + }, 60000); +}); diff --git a/cli/package-lock.json b/cli/package-lock.json index 7ed20a78f4..f7e349ba90 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -7,8 +7,12 @@ "": { "name": "@commonlyai/cli", "version": "0.1.155", + "bundleDependencies": [ + "@commonlyai/seats" + ], "license": "Apache-2.0", "dependencies": { + "@commonlyai/seats": "file:../packages/seats", "commander": "^12.0.0" }, "bin": { @@ -22,6 +26,14 @@ "node": ">=20.0.0" } }, + "../packages/seats": { + "name": "@commonlyai/seats", + "version": "0.0.0", + "license": "Apache-2.0", + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -518,6 +530,10 @@ "dev": true, "license": "MIT" }, + "node_modules/@commonlyai/seats": { + "resolved": "../packages/seats", + "link": true + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", diff --git a/cli/package.json b/cli/package.json index 2145947183..883171eb3a 100644 --- a/cli/package.json +++ b/cli/package.json @@ -2,7 +2,7 @@ "name": "@commonlyai/cli", "version": "0.1.155", "license": "Apache-2.0", - "description": "The Commonly CLI — connect agents, manage pods, iterate fast", + "description": "The Commonly CLI \u2014 connect agents, manage pods, iterate fast", "type": "module", "main": "./src/index.js", "bin": { @@ -32,11 +32,16 @@ "lint:fix": "eslint src --ext .js --fix", "test": "node --experimental-vm-modules node_modules/.bin/jest", "test:temp-cleanup": "node scripts/verify-temp-cleanup.mjs", + "prepack": "node scripts/require-bundled-seats.mjs", "prepublishOnly": "node -e \"require('fs').copyFileSync('../docs/agents/skills/commonly/SKILL.md','skills/commonly/SKILL.md')\"" }, "dependencies": { + "@commonlyai/seats": "file:../packages/seats", "commander": "^12.0.0" }, + "bundleDependencies": [ + "@commonlyai/seats" + ], "devDependencies": { "eslint": "^8.56.0", "jest": "^29.7.0" diff --git a/cli/scripts/require-bundled-seats.mjs b/cli/scripts/require-bundled-seats.mjs new file mode 100644 index 0000000000..63aaaf4f76 --- /dev/null +++ b/cli/scripts/require-bundled-seats.mjs @@ -0,0 +1,11 @@ +// prepack: refuse to pack the CLI without its bundled runtime package. +// `npm pack` on a tree where npm ci has not run exits 0 and silently leaves +// node_modules/@commonlyai/seats out of the tarball, which would publish a CLI +// whose adapters cannot load. +import { existsSync } from 'fs'; + +const manifest = new URL('../node_modules/@commonlyai/seats/package.json', import.meta.url); +if (!existsSync(manifest)) { + console.error('@commonlyai/seats is not installed in cli/node_modules; run `npm ci` in cli/ before packing or publishing'); + process.exit(1); +} diff --git a/cli/src/lib/adapters/hermes.js b/cli/src/lib/adapters/hermes.js index 2c6906c620..8181bbe1f2 100644 --- a/cli/src/lib/adapters/hermes.js +++ b/cli/src/lib/adapters/hermes.js @@ -1,199 +1,18 @@ -import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; -/** Hermes 0.22.2 local CLI wrapper. Hermes owns provider I/O; this file only spawns it. */ -import { requireSeatModel, seatModelRequired } from './seat-model.js'; -import { spawn as childSpawn, spawnSync } from 'child_process'; -import { createHash } from 'crypto'; -import { mkdir } from 'fs/promises'; import { homedir } from 'os'; import { join } from 'path'; -import { StringDecoder } from 'string_decoder'; +import { createHermesAdapter } from '@commonlyai/seats/adapters/hermes'; import { buildMemoryPreamble } from '../memory-bridge.js'; import { effectiveSandboxTrust } from '../environment.js'; -import { adapterFailure, spawnCredentials } from '../upstream-refusal.js'; - -export const TESTED_HERMES_VERSION = '0.22.2'; -const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; -const VERSION_TIMEOUT_MS = 15000; -const MAX_STDERR_CHARS = 4096; -const MAX_LINE_CHARS = 1024 * 1024; - -const readVersion = (env, spawnSyncImpl = spawnSync) => { - let probe; - for (let attempt = 0; attempt < 2; attempt += 1) { - probe = spawnSyncImpl('hermes', ['--version'], { - encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS, - }); - if (probe.error?.code === 'ENOENT') return null; - if (probe.error || probe.status !== 0) continue; - const version = String(probe.stdout || '').match(/Hermes Agent v([^\s]+)/i)?.[1]; - if (version !== TESTED_HERMES_VERSION) { - throw new Error(`Hermes ${version || 'unknown'} is unsupported; this adapter requires ${TESTED_HERMES_VERSION}`); - } - return version; - } - const status = probe.status ?? 'none'; - const signal = probe.signal ?? 'none'; - const code = probe.error?.code ?? 'none'; - throw new Error(`Hermes version probe failed after 2 attempts (status=${status}, signal=${signal}, code=${code}, timeout=${VERSION_TIMEOUT_MS}ms); check the Hermes installation or startup log`); -}; - -export const readHermesJsonl = (line, state) => { - let event; - try { event = JSON.parse(line); } catch { throw new Error('Hermes emitted invalid JSONL'); } - if (state.result) throw new Error('Hermes emitted data after its terminal result'); - if (event?.type === 'system' && event.subtype === 'init') { - state.started = true; - } else if (event?.type === 'result') { - state.result = event; - } else if (!state.started) { - throw new Error('Hermes emitted an event before init'); - } -}; - -const runHermes = ({ args, prompt, cwd, env, timeoutMs, spawnImpl, credentials, onSpawn, controls = {} }) => new Promise((resolve, reject) => { - const child = (spawnImpl || childSpawn)('hermes', args, { - cwd, env, stdio: ['pipe', 'pipe', 'pipe'], - }); - observeSpawn(child, onSpawn, controls); - const state = { started: false, result: null }; - const decoder = new StringDecoder('utf8'); - let buffer = ''; - let stderr = ''; - let timedOut = false; - let settled = false; - let closed = false; - let stopping = false; - let killTimer; - const finish = (error, result) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) reject(error); else resolve(result); - }; - const stop = () => { - if (stopping || closed) return; - stopping = true; - child.kill('SIGTERM'); - killTimer = setTimeout(() => { - if (!closed) child.kill('SIGKILL'); - }, 5000); - }; - const failAndStop = (error) => { - stop(); - finish(error); - }; - const timer = setTimeout(() => { - timedOut = true; - stop(); - }, timeoutMs); - child.on('error', (error) => failAndStop(adapterFailure('Hermes', error?.message || 'spawn failed', { credentials }))); - child.stdout.on('data', (chunk) => { - if (settled) return; - buffer += decoder.write(chunk); - let newline = buffer.indexOf('\n'); - while (newline !== -1) { - if (newline > MAX_LINE_CHARS) { - failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); - return; - } - const line = buffer.slice(0, newline).trim(); - buffer = buffer.slice(newline + 1); - if (line) { - try { readHermesJsonl(line, state); } catch (error) { - failAndStop(error); - return; - } - } - newline = buffer.indexOf('\n'); - } - if (buffer.length > MAX_LINE_CHARS) { - failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); - } - }); - child.stdout.on('error', (error) => failAndStop(adapterFailure('Hermes stdout', error?.message || 'read failed', { credentials }))); - child.stderr.on('data', (chunk) => { - stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_CHARS); - }); - child.on('close', (code) => { - closed = true; - clearTimeout(killTimer); - if (settled) return; - if (timedOut) return finish(new Error(`Hermes timed out after ${timeoutMs}ms`)); - buffer += decoder.end(); - if (buffer.trim()) { - try { readHermesJsonl(buffer.trim(), state); } catch (error) { return finish(error); } - } - const result = state.result; - if (!result || !state.started) return finish(new Error('Hermes exited without a complete JSONL result')); - if (code !== 0 || result.exit_code !== 0) { - return finish(adapterFailure('Hermes', result.error || stderr || `exit ${code}`, { - credentials, exitCode: code, - })); - } - if (typeof result.text !== 'string' || !result.session_id) { - return finish(new Error('Hermes result omitted text or session_id')); - } - return finish(null, { text: result.text, newSessionId: result.session_id }); - }); - child.stdin.on('error', (error) => failAndStop(adapterFailure('Hermes stdin', error?.message || 'write failed', { credentials }))); - child.stdin.end(prompt); +import { seatModelRequired } from './seat-model.js'; + +export { TESTED_HERMES_VERSION, readHermesJsonl } from '@commonlyai/seats/adapters/hermes'; + +// Commonly's policy for the generic Hermes adapter: its memory preamble, its +// legacy sandbox-trust names, the fleet seat-model switch, and the profile +// directory seats have always used. +export default createHermesAdapter({ + memoryPreamble: buildMemoryPreamble, + effectiveTrust: effectiveSandboxTrust, + seatModelRequired, + profileRoot: join(homedir(), '.commonly', 'hermes', 'profiles'), }); - -export default { - get requiresModel() { return seatModelRequired(); }, - name: 'hermes', - runtimeType: 'hermes', - supportsProvider: false, - providerKeyEnv: 'OPENROUTER_API_KEY', - - async validateEnvironment(environment) { - if (environment?.provider !== undefined) { - throw new Error('Hermes adapter does not consume environment.provider; supply OPENROUTER_API_KEY in the host environment'); - } - if (environment?.mcp?.length) { - throw new Error('Hermes adapter does not yet consume environment.mcp declarations'); - } - if (effectiveSandboxTrust(environment?.sandbox?.trust) === 'public' - || (environment?.sandbox?.mode && environment.sandbox.mode !== 'none')) { - throw new Error('Hermes public or sandbox-declared seats are unsupported: no enforced confinement adapter exists'); - } - }, - - async detect() { - const version = readVersion(process.env); - return version ? { path: 'hermes', version } : null; - }, - - async spawn(prompt, ctx = {}) { - const seatModel = requireSeatModel(ctx.environment, 'hermes'); - await this.validateEnvironment(ctx.environment); - // Keep Hermes away from the operator's default profile, including any - // Nous Portal/Cloud credentials there. The wrapper provides the sole key. - const seat = `${ctx.agentName || ''}\0${ctx.cwd || ''}`; - const home = ctx._hermesHome || join(homedir(), '.commonly', 'hermes', 'profiles', - createHash('sha256').update(seat).digest('hex').slice(0, 24)); - await mkdir(home, { recursive: true, mode: 0o700 }); - const env = { ...(ctx.env || process.env), HERMES_HOME: home }; - if (!readVersion(env, ctx._spawnSyncImpl)) { - throw new Error('Hermes Agent 0.22.2 is not on PATH'); - } - if (!env.OPENROUTER_API_KEY) { - throw new Error('Supply the operator OPENROUTER_API_KEY through Secret Manager before running Hermes'); - } - const args = ['chat', '--oneshot', '--query-file', '-', '--format', 'stream-json', - '--provider', 'openrouter', '--source', 'tool', '--no-restore-cwd']; - if (seatModel) args.push('--model', seatModel); - if (ctx.sessionId) args.push('--resume', ctx.sessionId); - const fullPrompt = buildMemoryPreamble(prompt, ctx.memoryLongTerm, { - freshSession: !ctx.sessionId, - }); - const requestedTimeout = Number(ctx.timeoutMs); - const timeoutMs = Number.isFinite(requestedTimeout) && requestedTimeout > 0 - ? Math.min(requestedTimeout, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS; - return runHermes({ - args, prompt: fullPrompt, cwd: ctx.cwd, env, timeoutMs, - spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args, { modelFlag: '--model', valueFlags: ['--query-file', '--format', '--provider', '--source', '--resume'] }), - credentials: spawnCredentials(ctx, ['OPENROUTER_API_KEY']), - }); - }, -}; diff --git a/cli/src/lib/adapters/seat-model.js b/cli/src/lib/adapters/seat-model.js index fd9a0a7dd9..75c226d4a4 100644 --- a/cli/src/lib/adapters/seat-model.js +++ b/cli/src/lib/adapters/seat-model.js @@ -1,15 +1,8 @@ +import { requireSeatModel as requireDeclaredModel } from '@commonlyai/seats/seat-model'; + /** Fleet runners opt in; BYO users retain their own CLI's model default. */ export const seatModelRequired = () => process.env.COMMONLY_REQUIRE_SEAT_MODEL === '1'; -export const requireSeatModel = (environment, adapterName, required = seatModelRequired()) => { - const model = environment?.model; - if (typeof model !== 'string' || !model.trim()) { - if (!required) return undefined; - const error = new Error( - `${adapterName} requires environment.model; set the seat model with commonly agent config --model before running`, - ); - error.code = 'seat_model_missing'; - throw error; - } - return model.trim(); -}; +export const requireSeatModel = (environment, adapterName, required = seatModelRequired()) => ( + requireDeclaredModel(environment, adapterName, required) +); diff --git a/cli/src/lib/adapters/stub.js b/cli/src/lib/adapters/stub.js index 9cae043466..5e756a1b38 100644 --- a/cli/src/lib/adapters/stub.js +++ b/cli/src/lib/adapters/stub.js @@ -1,24 +1 @@ -/** - * Stub adapter — the no-op reference implementation of the ADR-005 adapter - * contract. Used by `commonly agent attach stub` and the Phase-1a test - * harness so the run loop can be reviewed end-to-end without a real CLI - * on PATH. - * - * Contract (ADR-005 §Adapter pattern): - * detect(): Promise<{ path, version } | null> - * spawn(prompt, ctx): Promise<{ text, newSessionId?, memorySummary? }> - * - * The adapter MUST be pure: input (argv + env + prompt) → output (text + optional - * next session-id + optional memory summary). No direct network, no direct API - * calls, no hidden state. The run loop handles all Commonly-facing I/O. - */ - -export default { - name: 'stub', - async detect() { - return { path: '(builtin)', version: '0.0.0' }; - }, - async spawn(prompt, _ctx) { - return { text: `(stub) received: ${String(prompt ?? '').slice(0, 200)}` }; - }, -}; +export { default } from '@commonlyai/seats/adapters/stub'; diff --git a/cli/src/lib/runtime-observation.js b/cli/src/lib/runtime-observation.js index 2dc35fbfcb..9fd70c5530 100644 --- a/cli/src/lib/runtime-observation.js +++ b/cli/src/lib/runtime-observation.js @@ -1,30 +1,6 @@ -// Read only adapter-owned control arguments. Callers exclude prompt/stdin and -// provide every value-taking option, so a flag-shaped model/session value is -// never reinterpreted as a second option. Raw arguments never leave the runner. -export const controlsFromArguments = (args, { modelFlag, effortFlag, valueFlags = [], codex = false }) => { - const controls = { model: null, effort: null }; - const flags = new Set([modelFlag, effortFlag, ...valueFlags].filter(Boolean)); - for (let index = 0; index < args.length; index += 1) { - const flag = args[index]; - if (!flags.has(flag)) continue; - const value = args[++index]; - if (flag === modelFlag) controls.model = value || null; - if (flag === effortFlag) controls.effort = value || null; - if (codex && flag === '-c' && value?.startsWith('model_reasoning_effort=')) { - try { controls.effort = JSON.parse(value.slice('model_reasoning_effort='.length)); } catch { /* unknown encoding */ } - } - } - return controls; -}; - -// Called at the child's successful OS spawn event, not when a requested -// configuration is saved or when a turn is merely queued. Never send argv. -export const observeSpawn = (child, onSpawn, controls) => { - if (typeof onSpawn !== 'function') return; - child.once('spawn', () => { - try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }); } catch { /* diagnostic only */ } - }); -}; +// controlsFromArguments and observeSpawn live in the seats runtime package; +// the reporter below posts to a Commonly instance, so it stays in the CLI. +export { controlsFromArguments, observeSpawn } from '@commonlyai/seats/runtime-observation'; export const createRuntimeReporter = ({ client, harness, detect }) => { let queue = Promise.resolve(); diff --git a/cli/src/lib/upstream-refusal.js b/cli/src/lib/upstream-refusal.js index 598edb8a51..f70cef6312 100644 --- a/cli/src/lib/upstream-refusal.js +++ b/cli/src/lib/upstream-refusal.js @@ -1,354 +1 @@ -/** - * TASK-096 — name a refused model route, instead of reporting the silence it - * causes. - * - * Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape - * that made a seat look broken for two and a half days): pi runs its own retry - * ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and - * EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only - * `type: "error"` events — of which this produces none — so the wrapper logged - * `no wrapper-post (empty output)` and the cause sat unread in the stream. - * - * Two rules govern the text that reaches a seat log: - * - * 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes - * only the shapes we thought of, and a 401 can echo the key back. So a body - * contributes one NAMED field or nothing at all. - * 2. Exact-match refusal for credentials. If the kept text contains a value - * this spawn was handed, the text is dropped — a known value, not a guessed - * shape (Wren 70877). - * - * Two limits follow from rule 2. They are the accepted cost of refusing guessed - * shapes, named here so they are not rediscovered as defects later (Vera 71082): - * - * a. The match is exact and against THIS spawn's values only, so a credential - * that arrives transformed — base64 inside a proxy error, percent-encoded in - * a URL echo — does not match and the text is kept. Matching transformed - * shapes instead would reintroduce the class rule 1 exists to avoid. - * b. The keep-list can still surface someone ELSE's secret. A third party's key - * relayed in an upstream error is not detectable here, because the one - * inventory this module has is the set of values it was itself handed. - * - * Scope: this reader was wired into the pi adapter only. TASK-103 extended it - * to codex and claude, which are NOT silent the way pi was: the claude adapter - * rejects on a non-zero exit and deliberately reports stdout with it, because a - * `-p` run writes quota conditions there; the codex adapter rejects on - * `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text - * they raised was the raw tail, unscrubbed by the rules above, and carried no - * HTTP status for the run loop to classify on. `scrubAdapterFailure` below is - * the shared reader they now both go through, so the two rules cannot drift - * apart between three adapters. - */ - -/** The kept detail is truncated, ellipsis included, to this many characters. */ -export const MAX_REFUSAL_DETAIL = 256; - -/** - * The status an adapter prefixes to its error text: `"429: {…}"` -> `429`. - * Null for anything that is not a 4xx/5xx, so a message that merely starts with - * a number ("4040 tokens over budget") is not read as a status. - * - * The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1 - * emits `"429: {"message":…}"` for a JSON body and `"502 …"` for an - * HTML one. Both forms are matched here; a parser that required the colon would - * have kept `502 …` as the whole "body" and then dropped it as non-JSON, - * reporting the status alone with no sign that the body was never read. - */ -const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/; - -export const statusFromError = (value) => { - const match = STATUS_PREFIX.exec(String(value ?? '')); - if (!match) return null; - const status = Number(match[1]); - return status >= 400 && status <= 599 ? status : null; -}; - -/** - * The body the adapter carried after its status prefix, or the whole text when - * there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`; - * `"502 "` -> ``. - */ -export const bodyFromError = (value) => { - const text = String(value ?? ''); - if (statusFromError(text) === null) return text.trim(); - return text.replace(STATUS_PREFIX, '').trim(); -}; - -/** - * Replace credential values this spawn was handed with a marker. - * - * This is the exact-match half, for text that is kept for its whole shape rather - * than filtered to one field — an adapter's stderr tail, say, whose diagnostic - * value is in the lines around the secret. Values shorter than eight characters - * are ignored: redacting a one-character "credential" would corrupt ordinary - * prose, and no issued credential is that short. - */ -export const redactKnownCredentials = (text, credentials = []) => { - let out = String(text ?? ''); - for (const value of credentials) { - if (typeof value !== 'string' || value.length < 8) continue; - out = out.split(value).join('[redacted]'); - } - return out; -}; - -/** - * The keep-list. Given the raw body an adapter captured, keep at most one named - * field of it and only when that field is a string: - * - * `error.message` first (the OpenAI-shaped error), then a top-level `message` - * (which is the shape LiteLLM's budget refusal actually uses — the motivating - * case; a keep-list that kept only `error.message` would name the status and - * still hide the budget line that started this row). - * - * Everything else — HTML, a non-JSON body, a JSON body with neither field, a - * body echoing a known credential — contributes nothing. Returns null rather - * than a partial guess. - */ -export const keptRefusalDetail = (raw, { credentials = [] } = {}) => { - let parsed; - try { - parsed = JSON.parse(String(raw ?? '')); - } catch { - return null; - } - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; - - const candidate = parsed?.error?.message ?? parsed?.message; - if (typeof candidate !== 'string') return null; - - const text = candidate.replace(/\s+/g, ' ').trim(); - if (!text) return null; - - const redacted = redactKnownCredentials(text, credentials); - // `[redacted]` appearing is the signal that the body echoed something this - // spawn was handed. The status is still worth keeping; the text is not. - if (redacted !== text) return null; - - return text.length > MAX_REFUSAL_DETAIL - ? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…` - : text; -}; - -/** - * Read a pi JSON stream and return the refusal it ended on, or null. - * - * `auto_retry_end` is the terminal word — it names the status and the body after - * the ladder has finished — so it wins over the per-attempt `message_end` - * events that precede it. A run that refused on its first attempt and never - * retried still reports through `message_end`. - */ -export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => { - let status = null; - let raw = null; - - for (const line of String(stdout ?? '').split('\n')) { - if (!line.trim()) continue; - let event; - try { - event = JSON.parse(line); - } catch { - continue; - } - - if (event?.type === 'auto_retry_end' && event.success === false) { - const candidate = event.finalError ?? event.errorMessage; - const attemptStatus = statusFromError(candidate); - if (attemptStatus) { - status = attemptStatus; - raw = bodyFromError(candidate); - } - continue; - } - - const message = event?.message; - if ( - event?.type === 'message_end' - && message?.role === 'assistant' - && message.stopReason === 'error' - && status === null - ) { - const attemptStatus = statusFromError(message.errorMessage); - if (attemptStatus) { - status = attemptStatus; - raw = bodyFromError(message.errorMessage); - } - } - } - - if (status === null) return null; - return { status, detail: keptRefusalDetail(raw, { credentials }) }; -}; - -/** - * Is this text STRUCTURED upstream data — a JSON object or an array? The gate for - * the whole-tail fallback below, deliberately NOT `kept === null`, which is true - * for five different reasons (Vera 71350). - * - * An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is - * structured data with fields this code does not know, which is exactly what rule - * 1 exists for, and treating it as prose shipped the whole body under exact-match - * alone. A JSON *string* is the opposite case and does NOT count — `"Too many - * requests"` is a whole-shape text with no fields to reduce, so it keeps its - * wording like any other prose. - */ -const isStructuredBody = (raw) => { - try { - const parsed = JSON.parse(String(raw ?? '')); - return parsed !== null && typeof parsed === 'object'; - } catch { - return false; - } -}; - -/** - * A 3-digit number is read as a status only when the tail NAMES it as one: - * `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | ` - * segment followed by a colon, which is the shape claude's `stderr | stdout` - * join produces for a body that leads with its status. - * - * A bare `\b(\d{3})\b` scan is not enough, and the false positives are not - * hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds` - * all read as 500, and `classifySpawnFailure` tests the status FIELD before the - * message — so a count misread as a status stands the fleet down on a refusal - * that never happened, which is worse than no status at all (Vera 71352). - */ -const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi; -// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is, -// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED -// line: the `1` in `/1.1` is a -// WORD character, so `HTTP` and `429` are not within four non-word characters of -// each other. It read as `status: null` — an unclassified runtime failure on the -// single most recognisable shape, which is the misclassification this row exists -// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap -// would start reading `/v1/models 500` as a status. -const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi; -const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g; - -/** The first 4xx/5xx one of those shapes names, or null. */ -const scanStatuses = (text, pattern) => { - for (const match of String(text ?? '').matchAll(pattern)) { - const value = Number(match[1]); - if (value >= 400 && value <= 599) return value; - } - return null; -}; - -/** - * Find the status AND the body boundary, because they are the same question. - * - * Anchoring the status to a ` | ` segment start is not enough on its own: the - * body has to be taken from just past that status too, or a mid-tail read - * isolates the status and then hands the WHOLE joined tail to the body rules, - * which no longer look like a JSON body and so fall through to "keep it all". - * That is the difference between dropping a credential-echoing body and shipping - * the part of it that exact-match cannot see. - * - * `STATUS_COLON` is the only shape that moves the boundary, because it is the - * same shape as the anchored prefix (`429: {body}`), just found at a segment - * boundary — claude's `stderr | stdout` join. A status that appears inside a - * body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status - * (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body. - */ -const readStatusAndBody = (text) => { - const raw = String(text ?? ''); - const prefixed = statusFromError(raw); - if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) }; - // Only the colon shape moves the boundary — see the note above. A status LINE - // names the status but its reason phrase is prose, so the body stays the whole - // text and the exact-match rule protects it. - for (const match of raw.matchAll(STATUS_COLON)) { - const value = Number(match[1]); - if (value >= 400 && value <= 599) { - return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') }; - } - } - const named = scanStatuses(raw, STATUS_NAMED); - return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw }; -}; - -/** - * The shared reader for an adapter's own failure tail (TASK-103). - * - * The two rules above are about a body an adapter captured, and they apply here - * too — but the tail is a different shape from pi's refusal, and the difference - * decides how each rule lands: - * - * - The tail is often NOT structured. Claude's `-p` mode writes its terminal - * condition as prose (`Claude usage limit reached. Your limit will reset at - * 11:40pm.`), and that line is the whole diagnostic value of reporting - * stdout at all — 361 consecutive failures on 2026-08-03 carried no reason - * because of it. So the keep-list applies only when the tail parses as a - * body; when it does not, the text is KEPT as a whole shape and the - * exact-match rule is what protects it (`redactKnownCredentials`, which - * substitutes rather than drops, precisely because a whole-shape text has no - * second field to fall back to). - * - * - A body that IS structured (a JSON object or array) is reduced to one named - * field, and a body whose field echoed a value this spawn was handed is - * DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the - * whole tail is gated on the SHAPE (is it structured at all), never on - * "the keep-list returned null": that null has - * four other causes, and one of them is the credential echo this line exists - * to drop. Gating on `kept === null` would emit the whole body for that case, - * which is precisely what rule 2 forbids — and exact-match cannot see a - * transformed shape (limit (a) in the header), so a key that arrived - * base64'd or percent-encoded would ship (Vera 71350/71351). - * - * Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null. - * The caller decides where the status goes; `classifySpawnFailure` reads it off - * `error.status` or out of the message text, so both a leading `429: …` prefix - * and an attached field classify the same way. `detail` is `''` when nothing of - * the body may be shown — the status is then the whole diagnostic. - */ -export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => { - const raw = String(text ?? '').trim(); - const { status, body } = readStatusAndBody(raw); - const kept = keptRefusalDetail(body, { credentials }); - const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials)); - return { status, detail: detail.slice(0, limit) }; -}; - -/** - * The Error an adapter raises, built from its own failure tail. - * - * `status` is attached as a field, not only written into the text: the run - * loop's classifier tests `error.status === 429` before it looks at the - * message, and a refusal that is only readable by regex is one wording away - * from the misclassification that cost an hour on 2026-08-18 (see - * spawn-retry.js). The message still names the status so a human log says the - * same thing as the classifier's decision. - */ -export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => { - const { status, detail } = scrubAdapterFailure(text, { credentials, limit }); - const code = exitCode === null ? '' : ` exited with code ${exitCode}`; - const named = status === null ? '' : ` (upstream ${status})`; - const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`); - if (status !== null) error.status = status; - return error; -}; - -/** - * The seat-log clause. The status is always present; the detail only when the - * keep-list kept one. `grep 'upstream refused'` finds every refusal, which is - * the property the old `no wrapper-post (empty output)` line did not have. - */ -export const describeUpstreamRefusal = (refusal) => { - if (!refusal || typeof refusal.status !== 'number') return null; - return refusal.detail - ? `upstream refused ${refusal.status}: ${refusal.detail}` - : `upstream refused ${refusal.status}`; -}; - -/** - * The credential values this spawn was handed, for the exact-match check in - * `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the - * caller supplies them because only the adapter knows which key its route uses. - */ -export const spawnCredentials = (ctx = {}, envNames = []) => { - const env = ctx.env || process.env || {}; - return [ - ctx.runtimeToken, - ...envNames.map((name) => env[name]), - ].filter((value) => typeof value === 'string' && value.length >= 8); -}; +export * from '@commonlyai/seats/upstream-refusal'; diff --git a/packages/seats/README.md b/packages/seats/README.md new file mode 100644 index 0000000000..04e57fd9af --- /dev/null +++ b/packages/seats/README.md @@ -0,0 +1,9 @@ +# packages/seats + +The adapters that run an agent CLI as a supervised seat: one prompt in, one CLI run, one reply out. The package holds the contract, the registry, the failure naming, the spawn observation, and the first adapters (`stub`, `hermes`). + +It is private and is not published on its own. `@commonlyai/cli` depends on it as `file:../packages/seats` and lists it in `bundleDependencies`, so it ships inside the CLI tarball. + +**Rule:** nothing here imports Commonly code. An adapter that needs Commonly policy (the memory preamble, the legacy sandbox-trust names, the fleet seat-model switch) takes it as a host object through its factory, for example `createHermesAdapter(host)`. The CLI binds that host in `cli/src/lib/adapters/.js`. `cli/__tests__/seats-package.test.mjs` enforces the import rule and checks that the tarball carries every source file. + +The extraction plan is D62, extract-in-place. diff --git a/packages/seats/package.json b/packages/seats/package.json new file mode 100644 index 0000000000..f42ec21ff4 --- /dev/null +++ b/packages/seats/package.json @@ -0,0 +1,19 @@ +{ + "name": "@commonlyai/seats", + "version": "0.0.0", + "private": true, + "license": "Apache-2.0", + "description": "Adapters that run agent CLIs as supervised seats. Bundled inside @commonlyai/cli; not published on its own.", + "type": "module", + "exports": { + ".": "./src/index.js", + "./*": "./src/*.js" + }, + "files": [ + "src", + "README.md" + ], + "engines": { + "node": ">=20.0.0" + } +} diff --git a/packages/seats/src/adapters/hermes.js b/packages/seats/src/adapters/hermes.js new file mode 100644 index 0000000000..e54973f88f --- /dev/null +++ b/packages/seats/src/adapters/hermes.js @@ -0,0 +1,206 @@ +/** Hermes 0.22.2 local CLI wrapper. Hermes owns provider I/O; this file only spawns it. */ +import { spawn as childSpawn, spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import { mkdir } from 'fs/promises'; +import { homedir } from 'os'; +import { join } from 'path'; +import { StringDecoder } from 'string_decoder'; +import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { requireSeatModel } from '../seat-model.js'; +import { adapterFailure, spawnCredentials } from '../upstream-refusal.js'; + +export const TESTED_HERMES_VERSION = '0.22.2'; +const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; +const VERSION_TIMEOUT_MS = 15000; +const MAX_STDERR_CHARS = 4096; +const MAX_LINE_CHARS = 1024 * 1024; + +const readVersion = (env, spawnSyncImpl = spawnSync) => { + let probe; + for (let attempt = 0; attempt < 2; attempt += 1) { + probe = spawnSyncImpl('hermes', ['--version'], { + encoding: 'utf8', env, timeout: VERSION_TIMEOUT_MS, + }); + if (probe.error?.code === 'ENOENT') return null; + if (probe.error || probe.status !== 0) continue; + const version = String(probe.stdout || '').match(/Hermes Agent v([^\s]+)/i)?.[1]; + if (version !== TESTED_HERMES_VERSION) { + throw new Error(`Hermes ${version || 'unknown'} is unsupported; this adapter requires ${TESTED_HERMES_VERSION}`); + } + return version; + } + const status = probe.status ?? 'none'; + const signal = probe.signal ?? 'none'; + const code = probe.error?.code ?? 'none'; + throw new Error(`Hermes version probe failed after 2 attempts (status=${status}, signal=${signal}, code=${code}, timeout=${VERSION_TIMEOUT_MS}ms); check the Hermes installation or startup log`); +}; + +export const readHermesJsonl = (line, state) => { + let event; + try { event = JSON.parse(line); } catch { throw new Error('Hermes emitted invalid JSONL'); } + if (state.result) throw new Error('Hermes emitted data after its terminal result'); + if (event?.type === 'system' && event.subtype === 'init') { + state.started = true; + } else if (event?.type === 'result') { + state.result = event; + } else if (!state.started) { + throw new Error('Hermes emitted an event before init'); + } +}; + +const runHermes = ({ args, prompt, cwd, env, timeoutMs, spawnImpl, credentials, onSpawn, controls = {} }) => new Promise((resolve, reject) => { + const child = (spawnImpl || childSpawn)('hermes', args, { + cwd, env, stdio: ['pipe', 'pipe', 'pipe'], + }); + observeSpawn(child, onSpawn, controls); + const state = { started: false, result: null }; + const decoder = new StringDecoder('utf8'); + let buffer = ''; + let stderr = ''; + let timedOut = false; + let settled = false; + let closed = false; + let stopping = false; + let killTimer; + const finish = (error, result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); else resolve(result); + }; + const stop = () => { + if (stopping || closed) return; + stopping = true; + child.kill('SIGTERM'); + killTimer = setTimeout(() => { + if (!closed) child.kill('SIGKILL'); + }, 5000); + }; + const failAndStop = (error) => { + stop(); + finish(error); + }; + const timer = setTimeout(() => { + timedOut = true; + stop(); + }, timeoutMs); + child.on('error', (error) => failAndStop(adapterFailure('Hermes', error?.message || 'spawn failed', { credentials }))); + child.stdout.on('data', (chunk) => { + if (settled) return; + buffer += decoder.write(chunk); + let newline = buffer.indexOf('\n'); + while (newline !== -1) { + if (newline > MAX_LINE_CHARS) { + failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); + return; + } + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (line) { + try { readHermesJsonl(line, state); } catch (error) { + failAndStop(error); + return; + } + } + newline = buffer.indexOf('\n'); + } + if (buffer.length > MAX_LINE_CHARS) { + failAndStop(new Error('Hermes JSONL line exceeded 1 MiB')); + } + }); + child.stdout.on('error', (error) => failAndStop(adapterFailure('Hermes stdout', error?.message || 'read failed', { credentials }))); + child.stderr.on('data', (chunk) => { + stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_CHARS); + }); + child.on('close', (code) => { + closed = true; + clearTimeout(killTimer); + if (settled) return; + if (timedOut) return finish(new Error(`Hermes timed out after ${timeoutMs}ms`)); + buffer += decoder.end(); + if (buffer.trim()) { + try { readHermesJsonl(buffer.trim(), state); } catch (error) { return finish(error); } + } + const result = state.result; + if (!result || !state.started) return finish(new Error('Hermes exited without a complete JSONL result')); + if (code !== 0 || result.exit_code !== 0) { + return finish(adapterFailure('Hermes', result.error || stderr || `exit ${code}`, { + credentials, exitCode: code, + })); + } + if (typeof result.text !== 'string' || !result.session_id) { + return finish(new Error('Hermes result omitted text or session_id')); + } + return finish(null, { text: result.text, newSessionId: result.session_id }); + }); + child.stdin.on('error', (error) => failAndStop(adapterFailure('Hermes stdin', error?.message || 'write failed', { credentials }))); + child.stdin.end(prompt); +}); + +/** + * @param {import('../contract.js').SeatHost & { profileRoot?: string }} [host] + * profileRoot is where each seat's isolated HERMES_HOME is created. + */ +export const createHermesAdapter = ({ + memoryPreamble = (prompt) => prompt, + effectiveTrust = (trust) => trust, + seatModelRequired = () => false, + profileRoot = join(homedir(), '.seats', 'hermes', 'profiles'), +} = {}) => ({ + get requiresModel() { return seatModelRequired(); }, + name: 'hermes', + runtimeType: 'hermes', + supportsProvider: false, + providerKeyEnv: 'OPENROUTER_API_KEY', + + async validateEnvironment(environment) { + if (environment?.provider !== undefined) { + throw new Error('Hermes adapter does not consume environment.provider; supply OPENROUTER_API_KEY in the host environment'); + } + if (environment?.mcp?.length) { + throw new Error('Hermes adapter does not yet consume environment.mcp declarations'); + } + if (effectiveTrust(environment?.sandbox?.trust) === 'public' + || (environment?.sandbox?.mode && environment.sandbox.mode !== 'none')) { + throw new Error('Hermes public or sandbox-declared seats are unsupported: no enforced confinement adapter exists'); + } + }, + + async detect() { + const version = readVersion(process.env); + return version ? { path: 'hermes', version } : null; + }, + + async spawn(prompt, ctx = {}) { + const seatModel = requireSeatModel(ctx.environment, 'hermes', seatModelRequired()); + await this.validateEnvironment(ctx.environment); + // Keep Hermes away from the operator's default profile, including any + // Nous Portal/Cloud credentials there. The wrapper provides the sole key. + const seat = `${ctx.agentName || ''}\0${ctx.cwd || ''}`; + const home = ctx._hermesHome || join(profileRoot, + createHash('sha256').update(seat).digest('hex').slice(0, 24)); + await mkdir(home, { recursive: true, mode: 0o700 }); + const env = { ...(ctx.env || process.env), HERMES_HOME: home }; + if (!readVersion(env, ctx._spawnSyncImpl)) { + throw new Error('Hermes Agent 0.22.2 is not on PATH'); + } + if (!env.OPENROUTER_API_KEY) { + throw new Error('Supply the operator OPENROUTER_API_KEY through Secret Manager before running Hermes'); + } + const args = ['chat', '--oneshot', '--query-file', '-', '--format', 'stream-json', + '--provider', 'openrouter', '--source', 'tool', '--no-restore-cwd']; + if (seatModel) args.push('--model', seatModel); + if (ctx.sessionId) args.push('--resume', ctx.sessionId); + const fullPrompt = memoryPreamble(prompt, ctx.memoryLongTerm, { + freshSession: !ctx.sessionId, + }); + const requestedTimeout = Number(ctx.timeoutMs); + const timeoutMs = Number.isFinite(requestedTimeout) && requestedTimeout > 0 + ? Math.min(requestedTimeout, DEFAULT_TIMEOUT_MS) : DEFAULT_TIMEOUT_MS; + return runHermes({ + args, prompt: fullPrompt, cwd: ctx.cwd, env, timeoutMs, + spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args, { modelFlag: '--model', valueFlags: ['--query-file', '--format', '--provider', '--source', '--resume'] }), + credentials: spawnCredentials(ctx, ['OPENROUTER_API_KEY']), + }); + }, +}); diff --git a/packages/seats/src/adapters/stub.js b/packages/seats/src/adapters/stub.js new file mode 100644 index 0000000000..9cae043466 --- /dev/null +++ b/packages/seats/src/adapters/stub.js @@ -0,0 +1,24 @@ +/** + * Stub adapter — the no-op reference implementation of the ADR-005 adapter + * contract. Used by `commonly agent attach stub` and the Phase-1a test + * harness so the run loop can be reviewed end-to-end without a real CLI + * on PATH. + * + * Contract (ADR-005 §Adapter pattern): + * detect(): Promise<{ path, version } | null> + * spawn(prompt, ctx): Promise<{ text, newSessionId?, memorySummary? }> + * + * The adapter MUST be pure: input (argv + env + prompt) → output (text + optional + * next session-id + optional memory summary). No direct network, no direct API + * calls, no hidden state. The run loop handles all Commonly-facing I/O. + */ + +export default { + name: 'stub', + async detect() { + return { path: '(builtin)', version: '0.0.0' }; + }, + async spawn(prompt, _ctx) { + return { text: `(stub) received: ${String(prompt ?? '').slice(0, 200)}` }; + }, +}; diff --git a/packages/seats/src/contract.js b/packages/seats/src/contract.js new file mode 100644 index 0000000000..3980951746 --- /dev/null +++ b/packages/seats/src/contract.js @@ -0,0 +1,48 @@ +/** + * The adapter contract (ADR-005 §Adapter pattern). An adapter turns one prompt + * into one CLI run and one reply. It owns argv, env and output parsing for its + * CLI; the caller owns everything that talks to a Commonly instance. + * + * @typedef {object} Adapter + * @property {string} name + * @property {string} [runtimeType] + * @property {string} [providerKeyEnv] host env var the adapter's provider key arrives in + * @property {() => Promise<{ path: string, version: string } | null>} detect + * @property {(environment: object) => Promise} [validateEnvironment] refuse before spawn + * @property {(prompt: string, ctx: object) => Promise} spawn + * + * @typedef {object} SpawnResult + * @property {string} text + * @property {string} [newSessionId] + * @property {number} [contextTokens] + * @property {string} [memorySummary] + * + * Adapters that need host policy are built by a factory that takes a SeatHost. + * Every field is optional; an absent field takes the neutral default named. + * + * @typedef {object} SeatHost + * @property {(prompt: string, longTerm: string | null | undefined, opts: { freshSession: boolean }) => string} [memoryPreamble] default: the prompt unchanged + * @property {(trust: string | undefined) => string | undefined} [effectiveTrust] default: the declared trust unchanged + * @property {() => boolean} [seatModelRequired] default: false + */ + +/** + * @param {Adapter[]} adapters + */ +export const createRegistry = (adapters) => { + const byName = {}; + for (const adapter of adapters) { + if (!adapter?.name) throw new Error('every adapter needs a name'); + if (byName[adapter.name]) throw new Error(`duplicate adapter name: ${adapter.name}`); + byName[adapter.name] = adapter; + } + return { + adapters: byName, + names: () => Object.keys(byName), + get: (name) => byName[name] || null, + }; +}; + +export const providerKeyEnvNames = (adapters) => [...new Set( + Object.values(adapters).map((adapter) => adapter.providerKeyEnv).filter(Boolean), +)]; diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js new file mode 100644 index 0000000000..7667aefdb9 --- /dev/null +++ b/packages/seats/src/index.js @@ -0,0 +1,6 @@ +export { createRegistry, providerKeyEnvNames } from './contract.js'; +export { requireSeatModel } from './seat-model.js'; +export { controlsFromArguments, observeSpawn } from './runtime-observation.js'; +export { adapterFailure, spawnCredentials } from './upstream-refusal.js'; +export { default as stub } from './adapters/stub.js'; +export { createHermesAdapter } from './adapters/hermes.js'; diff --git a/packages/seats/src/runtime-observation.js b/packages/seats/src/runtime-observation.js new file mode 100644 index 0000000000..59eb63ee8c --- /dev/null +++ b/packages/seats/src/runtime-observation.js @@ -0,0 +1,27 @@ +// Read only adapter-owned control arguments. Callers exclude prompt/stdin and +// provide every value-taking option, so a flag-shaped model/session value is +// never reinterpreted as a second option. Raw arguments never leave the runner. +export const controlsFromArguments = (args, { modelFlag, effortFlag, valueFlags = [], codex = false }) => { + const controls = { model: null, effort: null }; + const flags = new Set([modelFlag, effortFlag, ...valueFlags].filter(Boolean)); + for (let index = 0; index < args.length; index += 1) { + const flag = args[index]; + if (!flags.has(flag)) continue; + const value = args[++index]; + if (flag === modelFlag) controls.model = value || null; + if (flag === effortFlag) controls.effort = value || null; + if (codex && flag === '-c' && value?.startsWith('model_reasoning_effort=')) { + try { controls.effort = JSON.parse(value.slice('model_reasoning_effort='.length)); } catch { /* unknown encoding */ } + } + } + return controls; +}; + +// Called at the child's successful OS spawn event, not when a requested +// configuration is saved or when a turn is merely queued. Never send argv. +export const observeSpawn = (child, onSpawn, controls) => { + if (typeof onSpawn !== 'function') return; + child.once('spawn', () => { + try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }); } catch { /* diagnostic only */ } + }); +}; diff --git a/packages/seats/src/seat-model.js b/packages/seats/src/seat-model.js new file mode 100644 index 0000000000..b672e394d2 --- /dev/null +++ b/packages/seats/src/seat-model.js @@ -0,0 +1,17 @@ +/** + * The model a seat runs on is a declared field (`environment.model`), never a + * CLI default the operator did not choose. Whether a missing model is an error + * is the host's policy, passed as `required`. + */ +export const requireSeatModel = (environment, adapterName, required = false) => { + const model = environment?.model; + if (typeof model !== 'string' || !model.trim()) { + if (!required) return undefined; + const error = new Error( + `${adapterName} requires environment.model; set the seat model with commonly agent config --model before running`, + ); + error.code = 'seat_model_missing'; + throw error; + } + return model.trim(); +}; diff --git a/packages/seats/src/upstream-refusal.js b/packages/seats/src/upstream-refusal.js new file mode 100644 index 0000000000..598edb8a51 --- /dev/null +++ b/packages/seats/src/upstream-refusal.js @@ -0,0 +1,354 @@ +/** + * TASK-096 — name a refused model route, instead of reporting the silence it + * causes. + * + * Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape + * that made a seat look broken for two and a half days): pi runs its own retry + * ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and + * EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only + * `type: "error"` events — of which this produces none — so the wrapper logged + * `no wrapper-post (empty output)` and the cause sat unread in the stream. + * + * Two rules govern the text that reaches a seat log: + * + * 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes + * only the shapes we thought of, and a 401 can echo the key back. So a body + * contributes one NAMED field or nothing at all. + * 2. Exact-match refusal for credentials. If the kept text contains a value + * this spawn was handed, the text is dropped — a known value, not a guessed + * shape (Wren 70877). + * + * Two limits follow from rule 2. They are the accepted cost of refusing guessed + * shapes, named here so they are not rediscovered as defects later (Vera 71082): + * + * a. The match is exact and against THIS spawn's values only, so a credential + * that arrives transformed — base64 inside a proxy error, percent-encoded in + * a URL echo — does not match and the text is kept. Matching transformed + * shapes instead would reintroduce the class rule 1 exists to avoid. + * b. The keep-list can still surface someone ELSE's secret. A third party's key + * relayed in an upstream error is not detectable here, because the one + * inventory this module has is the set of values it was itself handed. + * + * Scope: this reader was wired into the pi adapter only. TASK-103 extended it + * to codex and claude, which are NOT silent the way pi was: the claude adapter + * rejects on a non-zero exit and deliberately reports stdout with it, because a + * `-p` run writes quota conditions there; the codex adapter rejects on + * `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text + * they raised was the raw tail, unscrubbed by the rules above, and carried no + * HTTP status for the run loop to classify on. `scrubAdapterFailure` below is + * the shared reader they now both go through, so the two rules cannot drift + * apart between three adapters. + */ + +/** The kept detail is truncated, ellipsis included, to this many characters. */ +export const MAX_REFUSAL_DETAIL = 256; + +/** + * The status an adapter prefixes to its error text: `"429: {…}"` -> `429`. + * Null for anything that is not a 4xx/5xx, so a message that merely starts with + * a number ("4040 tokens over budget") is not read as a status. + * + * The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1 + * emits `"429: {"message":…}"` for a JSON body and `"502 …"` for an + * HTML one. Both forms are matched here; a parser that required the colon would + * have kept `502 …` as the whole "body" and then dropped it as non-JSON, + * reporting the status alone with no sign that the body was never read. + */ +const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/; + +export const statusFromError = (value) => { + const match = STATUS_PREFIX.exec(String(value ?? '')); + if (!match) return null; + const status = Number(match[1]); + return status >= 400 && status <= 599 ? status : null; +}; + +/** + * The body the adapter carried after its status prefix, or the whole text when + * there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`; + * `"502 "` -> ``. + */ +export const bodyFromError = (value) => { + const text = String(value ?? ''); + if (statusFromError(text) === null) return text.trim(); + return text.replace(STATUS_PREFIX, '').trim(); +}; + +/** + * Replace credential values this spawn was handed with a marker. + * + * This is the exact-match half, for text that is kept for its whole shape rather + * than filtered to one field — an adapter's stderr tail, say, whose diagnostic + * value is in the lines around the secret. Values shorter than eight characters + * are ignored: redacting a one-character "credential" would corrupt ordinary + * prose, and no issued credential is that short. + */ +export const redactKnownCredentials = (text, credentials = []) => { + let out = String(text ?? ''); + for (const value of credentials) { + if (typeof value !== 'string' || value.length < 8) continue; + out = out.split(value).join('[redacted]'); + } + return out; +}; + +/** + * The keep-list. Given the raw body an adapter captured, keep at most one named + * field of it and only when that field is a string: + * + * `error.message` first (the OpenAI-shaped error), then a top-level `message` + * (which is the shape LiteLLM's budget refusal actually uses — the motivating + * case; a keep-list that kept only `error.message` would name the status and + * still hide the budget line that started this row). + * + * Everything else — HTML, a non-JSON body, a JSON body with neither field, a + * body echoing a known credential — contributes nothing. Returns null rather + * than a partial guess. + */ +export const keptRefusalDetail = (raw, { credentials = [] } = {}) => { + let parsed; + try { + parsed = JSON.parse(String(raw ?? '')); + } catch { + return null; + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + + const candidate = parsed?.error?.message ?? parsed?.message; + if (typeof candidate !== 'string') return null; + + const text = candidate.replace(/\s+/g, ' ').trim(); + if (!text) return null; + + const redacted = redactKnownCredentials(text, credentials); + // `[redacted]` appearing is the signal that the body echoed something this + // spawn was handed. The status is still worth keeping; the text is not. + if (redacted !== text) return null; + + return text.length > MAX_REFUSAL_DETAIL + ? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…` + : text; +}; + +/** + * Read a pi JSON stream and return the refusal it ended on, or null. + * + * `auto_retry_end` is the terminal word — it names the status and the body after + * the ladder has finished — so it wins over the per-attempt `message_end` + * events that precede it. A run that refused on its first attempt and never + * retried still reports through `message_end`. + */ +export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => { + let status = null; + let raw = null; + + for (const line of String(stdout ?? '').split('\n')) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + + if (event?.type === 'auto_retry_end' && event.success === false) { + const candidate = event.finalError ?? event.errorMessage; + const attemptStatus = statusFromError(candidate); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(candidate); + } + continue; + } + + const message = event?.message; + if ( + event?.type === 'message_end' + && message?.role === 'assistant' + && message.stopReason === 'error' + && status === null + ) { + const attemptStatus = statusFromError(message.errorMessage); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(message.errorMessage); + } + } + } + + if (status === null) return null; + return { status, detail: keptRefusalDetail(raw, { credentials }) }; +}; + +/** + * Is this text STRUCTURED upstream data — a JSON object or an array? The gate for + * the whole-tail fallback below, deliberately NOT `kept === null`, which is true + * for five different reasons (Vera 71350). + * + * An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is + * structured data with fields this code does not know, which is exactly what rule + * 1 exists for, and treating it as prose shipped the whole body under exact-match + * alone. A JSON *string* is the opposite case and does NOT count — `"Too many + * requests"` is a whole-shape text with no fields to reduce, so it keeps its + * wording like any other prose. + */ +const isStructuredBody = (raw) => { + try { + const parsed = JSON.parse(String(raw ?? '')); + return parsed !== null && typeof parsed === 'object'; + } catch { + return false; + } +}; + +/** + * A 3-digit number is read as a status only when the tail NAMES it as one: + * `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | ` + * segment followed by a colon, which is the shape claude's `stderr | stdout` + * join produces for a body that leads with its status. + * + * A bare `\b(\d{3})\b` scan is not enough, and the false positives are not + * hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds` + * all read as 500, and `classifySpawnFailure` tests the status FIELD before the + * message — so a count misread as a status stands the fleet down on a refusal + * that never happened, which is worse than no status at all (Vera 71352). + */ +const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi; +// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is, +// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED +// line: the `1` in `/1.1` is a +// WORD character, so `HTTP` and `429` are not within four non-word characters of +// each other. It read as `status: null` — an unclassified runtime failure on the +// single most recognisable shape, which is the misclassification this row exists +// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap +// would start reading `/v1/models 500` as a status. +const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi; +const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g; + +/** The first 4xx/5xx one of those shapes names, or null. */ +const scanStatuses = (text, pattern) => { + for (const match of String(text ?? '').matchAll(pattern)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) return value; + } + return null; +}; + +/** + * Find the status AND the body boundary, because they are the same question. + * + * Anchoring the status to a ` | ` segment start is not enough on its own: the + * body has to be taken from just past that status too, or a mid-tail read + * isolates the status and then hands the WHOLE joined tail to the body rules, + * which no longer look like a JSON body and so fall through to "keep it all". + * That is the difference between dropping a credential-echoing body and shipping + * the part of it that exact-match cannot see. + * + * `STATUS_COLON` is the only shape that moves the boundary, because it is the + * same shape as the anchored prefix (`429: {body}`), just found at a segment + * boundary — claude's `stderr | stdout` join. A status that appears inside a + * body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status + * (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body. + */ +const readStatusAndBody = (text) => { + const raw = String(text ?? ''); + const prefixed = statusFromError(raw); + if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) }; + // Only the colon shape moves the boundary — see the note above. A status LINE + // names the status but its reason phrase is prose, so the body stays the whole + // text and the exact-match rule protects it. + for (const match of raw.matchAll(STATUS_COLON)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) { + return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') }; + } + } + const named = scanStatuses(raw, STATUS_NAMED); + return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw }; +}; + +/** + * The shared reader for an adapter's own failure tail (TASK-103). + * + * The two rules above are about a body an adapter captured, and they apply here + * too — but the tail is a different shape from pi's refusal, and the difference + * decides how each rule lands: + * + * - The tail is often NOT structured. Claude's `-p` mode writes its terminal + * condition as prose (`Claude usage limit reached. Your limit will reset at + * 11:40pm.`), and that line is the whole diagnostic value of reporting + * stdout at all — 361 consecutive failures on 2026-08-03 carried no reason + * because of it. So the keep-list applies only when the tail parses as a + * body; when it does not, the text is KEPT as a whole shape and the + * exact-match rule is what protects it (`redactKnownCredentials`, which + * substitutes rather than drops, precisely because a whole-shape text has no + * second field to fall back to). + * + * - A body that IS structured (a JSON object or array) is reduced to one named + * field, and a body whose field echoed a value this spawn was handed is + * DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the + * whole tail is gated on the SHAPE (is it structured at all), never on + * "the keep-list returned null": that null has + * four other causes, and one of them is the credential echo this line exists + * to drop. Gating on `kept === null` would emit the whole body for that case, + * which is precisely what rule 2 forbids — and exact-match cannot see a + * transformed shape (limit (a) in the header), so a key that arrived + * base64'd or percent-encoded would ship (Vera 71350/71351). + * + * Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null. + * The caller decides where the status goes; `classifySpawnFailure` reads it off + * `error.status` or out of the message text, so both a leading `429: …` prefix + * and an attached field classify the same way. `detail` is `''` when nothing of + * the body may be shown — the status is then the whole diagnostic. + */ +export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => { + const raw = String(text ?? '').trim(); + const { status, body } = readStatusAndBody(raw); + const kept = keptRefusalDetail(body, { credentials }); + const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials)); + return { status, detail: detail.slice(0, limit) }; +}; + +/** + * The Error an adapter raises, built from its own failure tail. + * + * `status` is attached as a field, not only written into the text: the run + * loop's classifier tests `error.status === 429` before it looks at the + * message, and a refusal that is only readable by regex is one wording away + * from the misclassification that cost an hour on 2026-08-18 (see + * spawn-retry.js). The message still names the status so a human log says the + * same thing as the classifier's decision. + */ +export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => { + const { status, detail } = scrubAdapterFailure(text, { credentials, limit }); + const code = exitCode === null ? '' : ` exited with code ${exitCode}`; + const named = status === null ? '' : ` (upstream ${status})`; + const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`); + if (status !== null) error.status = status; + return error; +}; + +/** + * The seat-log clause. The status is always present; the detail only when the + * keep-list kept one. `grep 'upstream refused'` finds every refusal, which is + * the property the old `no wrapper-post (empty output)` line did not have. + */ +export const describeUpstreamRefusal = (refusal) => { + if (!refusal || typeof refusal.status !== 'number') return null; + return refusal.detail + ? `upstream refused ${refusal.status}: ${refusal.detail}` + : `upstream refused ${refusal.status}`; +}; + +/** + * The credential values this spawn was handed, for the exact-match check in + * `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the + * caller supplies them because only the adapter knows which key its route uses. + */ +export const spawnCredentials = (ctx = {}, envNames = []) => { + const env = ctx.env || process.env || {}; + return [ + ctx.runtimeToken, + ...envNames.map((name) => env[name]), + ].filter((value) => typeof value === 'string' && value.length >= 8); +}; From 1a047cf11594056036a16bf092a1d1378f4ace49 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:36:41 -0700 Subject: [PATCH 02/20] feat(seats): trace events, a trace store, named failures and a fake provider The shared seams for the supervisor, the generic ACP adapter, per-run credentials, the compatibility matrix and the dashboard: - trace/events.js: the event vocabulary, named after ACP's session/update kinds - trace/store.js: an append-only JSONL store (0600 files, 0700 dirs) - failures.js: one kind per way a run ends badly, each with a retry flag - test-support/fake-provider.js: a local OpenAI-compatible server so real CLIs run a genuine turn with no key and no network Tests run with node --test inside the package. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/package.json | 3 + packages/seats/src/failures.js | 38 +++++++++ packages/seats/src/index.js | 3 + packages/seats/src/trace/events.js | 33 ++++++++ packages/seats/src/trace/store.js | 74 +++++++++++++++++ packages/seats/test-support/fake-provider.js | 83 ++++++++++++++++++++ packages/seats/test/trace-store.test.js | 77 ++++++++++++++++++ 7 files changed, 311 insertions(+) create mode 100644 packages/seats/src/failures.js create mode 100644 packages/seats/src/trace/events.js create mode 100644 packages/seats/src/trace/store.js create mode 100644 packages/seats/test-support/fake-provider.js create mode 100644 packages/seats/test/trace-store.test.js diff --git a/packages/seats/package.json b/packages/seats/package.json index f42ec21ff4..ca4b81f327 100644 --- a/packages/seats/package.json +++ b/packages/seats/package.json @@ -15,5 +15,8 @@ ], "engines": { "node": ">=20.0.0" + }, + "scripts": { + "test": "node --test \"test/**/*.test.js\"" } } diff --git a/packages/seats/src/failures.js b/packages/seats/src/failures.js new file mode 100644 index 0000000000..046a39c852 --- /dev/null +++ b/packages/seats/src/failures.js @@ -0,0 +1,38 @@ +/** + * Named run failures. Every way a run can end badly has one kind, so a trace, + * a retry policy and a dashboard all read the same word. A kind is a contract: + * add one here, never mint one at a call site. + */ +import { scrubAdapterFailure } from './upstream-refusal.js'; + +export const FAILURE_KINDS = Object.freeze({ + stuck: { retryable: true, meaning: 'no progress signal within the liveness window' }, + timeout: { retryable: true, meaning: 'the run exceeded its wall-clock budget' }, + exited: { retryable: true, meaning: 'the CLI process ended before the turn settled' }, + protocol: { retryable: false, meaning: 'the CLI broke its wire protocol' }, + refusal: { retryable: false, meaning: 'the model or provider refused; never retried' }, + max_tokens: { retryable: false, meaning: 'the turn hit its token limit' }, + max_turn_requests: { retryable: false, meaning: 'the turn hit its request limit' }, + cancelled: { retryable: false, meaning: 'the run was cancelled' }, + credential_denied: { retryable: false, meaning: 'the run asked for a credential its grant does not allow' }, + unsupported: { retryable: false, meaning: 'the CLI or its version is outside what this adapter supports' }, +}); + +export class RunFailure extends Error { + constructor(kind, detail, { cause } = {}) { + if (!FAILURE_KINDS[kind]) throw new Error(`unknown failure kind: ${kind}`); + super(detail ? `${kind}: ${detail}` : kind, cause ? { cause } : undefined); + this.name = 'RunFailure'; + this.kind = kind; + this.retryable = FAILURE_KINDS[kind].retryable; + this.detail = detail || ''; + } +} + +/** Build a failure whose detail has known credentials and secret shapes scrubbed. */ +export const runFailure = (kind, text, { credentials = [], limit } = {}) => { + const { detail } = scrubAdapterFailure(String(text ?? ''), { credentials, limit }); + return new RunFailure(kind, detail); +}; + +export const failureKindOf = (error) => (error instanceof RunFailure ? error.kind : null); diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index 7667aefdb9..08c968faeb 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -4,3 +4,6 @@ export { controlsFromArguments, observeSpawn } from './runtime-observation.js'; export { adapterFailure, spawnCredentials } from './upstream-refusal.js'; export { default as stub } from './adapters/stub.js'; export { createHermesAdapter } from './adapters/hermes.js'; +export { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; +export { EVENT_TYPES, isEventType, traceOf, NULL_TRACE } from './trace/events.js'; +export { createTraceStore } from './trace/store.js'; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js new file mode 100644 index 0000000000..c78b0633d5 --- /dev/null +++ b/packages/seats/src/trace/events.js @@ -0,0 +1,33 @@ +/** + * Trace event vocabulary. Names follow ACP's session/update kinds where one + * exists, so an ACP agent's stream maps across without a translation table. + * + * Event: { runId, seq, ts, type, data }. `seq` is per run, starting at 1; + * `ts` is an ISO string. Events never carry prompt text, credentials or + * environment values — only names, ids, sizes and the agent's own output. + */ +export const EVENT_TYPES = Object.freeze([ + 'run.started', // { adapter, cli: {name, version}?, sessionId?, attempt } + 'run.settled', // { outcome: 'ok'|'failed'|'stuck'|'cancelled', failure?: {kind, detail}, durationMs, attempts } + 'turn.prompt_sent', // { chars } + 'agent.message_chunk', // { text } + 'agent.thought_chunk', // { chars } thoughts are counted, not stored + 'tool.call', // { toolCallId, title, kind?, status? } + 'tool.update', // { toolCallId, status?, title? } + 'plan', // { entries: [{content, status}] } + 'permission.requested', // { toolCallId?, options: [{optionId, kind}] } + 'permission.decided', // { outcome: 'selected'|'cancelled', optionId?, kind? } + 'usage', // { inputTokens?, outputTokens?, totalTokens?, cost? } null-not-zero when unknown + 'liveness.stalled', // { idleMs, windowMs } + 'liveness.recovery', // { action: 'cancel'|'kill'|'retry', attempt } + 'credential.granted', // { names: [...] } names only, never values + 'credential.denied', // { name } + 'process.exited', // { code, signal } +]); + +const KNOWN = new Set(EVENT_TYPES); +export const isEventType = (type) => KNOWN.has(type); + +/** The trace hook an adapter receives as ctx.trace. Absent hook → no-op. */ +export const NULL_TRACE = Object.freeze({ emit() {} }); +export const traceOf = (ctx) => (ctx && ctx.trace && typeof ctx.trace.emit === 'function' ? ctx.trace : NULL_TRACE); diff --git a/packages/seats/src/trace/store.js b/packages/seats/src/trace/store.js new file mode 100644 index 0000000000..bacff1e8b1 --- /dev/null +++ b/packages/seats/src/trace/store.js @@ -0,0 +1,74 @@ +/** + * Append-only trace store on the local filesystem. + * + * /runs/.jsonl one event per line, in seq order + * /index.jsonl one line per started run, one per settled run + * + * Files are 0600 and directories 0700: a trace holds agent output. + */ +import { randomUUID } from 'node:crypto'; +import { appendFileSync, mkdirSync, readFileSync, existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { isEventType } from './events.js'; + +const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; +const readJsonl = (path) => (existsSync(path) + ? readFileSync(path, 'utf8').split('\n').filter(Boolean).flatMap((line) => { + try { return [JSON.parse(line)]; } catch { return []; } + }) + : []); + +export const createTraceStore = ({ root, now = () => new Date() } = {}) => { + if (!root) throw new Error('createTraceStore needs a root directory'); + const runsDir = join(root, 'runs'); + mkdirSync(runsDir, { recursive: true, mode: 0o700 }); + const append = (path, value) => appendFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o600 }); + const indexPath = join(root, 'index.jsonl'); + + const startRun = (meta = {}) => { + const runId = meta.runId || randomUUID(); + if (!RUN_ID.test(runId)) throw new Error('runId must be 6-64 characters of [A-Za-z0-9_-]'); + const path = join(runsDir, `${runId}.jsonl`); + try { + writeFileSync(path, '', { mode: 0o600, flag: 'wx' }); + } catch (error) { + if (error.code === 'EEXIST') throw new Error(`run ${runId} already exists`); + throw error; + } + let seq = 0; + let settled = false; + const startedAt = now(); + const emit = (type, data = {}) => { + if (!isEventType(type)) throw new Error(`unknown trace event type: ${type}`); + if (settled) return null; + seq += 1; + const event = { runId, seq, ts: now().toISOString(), type, data }; + append(path, event); + if (type === 'run.settled') { + settled = true; + append(indexPath, { runId, settledAt: event.ts, outcome: data.outcome, failure: data.failure || null, durationMs: data.durationMs ?? null }); + } + return event; + }; + append(indexPath, { runId, startedAt: startedAt.toISOString(), adapter: meta.adapter || null, label: meta.label || null }); + return { runId, emit, get settled() { return settled; } }; + }; + + const listRuns = ({ limit = 50 } = {}) => { + const byId = new Map(); + for (const line of readJsonl(indexPath)) { + const row = byId.get(line.runId) || { runId: line.runId, outcome: 'running' }; + Object.assign(row, line); + byId.set(line.runId, row); + } + return [...byId.values()].sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt))).slice(0, limit); + }; + + const readRun = (runId) => { + if (!RUN_ID.test(String(runId))) return null; + const path = join(runsDir, `${runId}.jsonl`); + return existsSync(path) ? readJsonl(path) : null; + }; + + return { root, startRun, listRuns, readRun }; +}; diff --git a/packages/seats/test-support/fake-provider.js b/packages/seats/test-support/fake-provider.js new file mode 100644 index 0000000000..62bb98bf77 --- /dev/null +++ b/packages/seats/test-support/fake-provider.js @@ -0,0 +1,83 @@ +/** + * A local OpenAI-compatible model server for tests and the compatibility + * matrix. Real CLIs (goose, opencode, hermes, …) point their provider base URL + * here and run a genuine turn with no key and no network. + * + * Behaviour is chosen per request by a directive in the last user message: + * [[reply:TEXT]] answer TEXT (default: PONG) + * [[stall]] accept the request and never answer (liveness tests) + * [[refuse]] answer HTTP 400 with a content-policy error body + * [[fail:CODE]] answer HTTP CODE + * Serves POST /v1/chat/completions (stream and non-stream) and GET /v1/models. + */ +import { createServer } from 'node:http'; + +const directiveOf = (body) => { + const messages = Array.isArray(body?.messages) ? body.messages : []; + const text = messages.filter((m) => m?.role === 'user').map((m) => ( + typeof m.content === 'string' ? m.content + : Array.isArray(m.content) ? m.content.map((p) => p?.text || '').join('') : '' + )).pop() || ''; + const match = /\[\[(reply|stall|refuse|fail)(?::([^\]]*))?\]\]/.exec(text); + return match ? { kind: match[1], arg: match[2] ?? '' } : { kind: 'reply', arg: '' }; +}; + +export const startFakeProvider = async ({ model = 'fake-model', defaultReply = 'PONG', port = 0 } = {}) => { + const requests = []; + const open = new Set(); + const server = createServer((req, res) => { + open.add(res); + res.on('close', () => open.delete(res)); + let raw = ''; + req.on('data', (chunk) => { raw += chunk; }); + req.on('end', () => { + if (req.method === 'GET' && /\/models$/.test(req.url)) { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ object: 'list', data: [{ id: model, object: 'model', owned_by: 'fake' }] })); + return; + } + if (req.method !== 'POST' || !/\/chat\/completions$/.test(req.url)) { + res.writeHead(404); res.end(); return; + } + let body = {}; + try { body = JSON.parse(raw || '{}'); } catch { /* answer as default */ } + const directive = directiveOf(body); + requests.push({ url: req.url, model: body.model, stream: !!body.stream, directive: directive.kind, authorization: req.headers.authorization ? 'present' : 'absent' }); + if (directive.kind === 'stall') return; // hold the socket open, never answer + if (directive.kind === 'refuse') { + res.writeHead(400, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'The request was rejected by the content policy.', type: 'invalid_request_error', code: 'content_policy_violation' } })); + return; + } + if (directive.kind === 'fail') { + res.writeHead(Number(directive.arg) || 500, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'fake provider failure', type: 'server_error' } })); + return; + } + const text = directive.arg || defaultReply; + const id = `chatcmpl-fake-${requests.length}`; + const created = Math.floor(Date.now() / 1000); + const usage = { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }; + if (body.stream) { + res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }); + const chunk = (delta, finish = null, extra = {}) => res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`); + chunk({ role: 'assistant', content: '' }); + chunk({ content: text }); + chunk({}, 'stop', body.stream_options?.include_usage ? {} : { usage }); + if (body.stream_options?.include_usage) res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [], usage })}\n\n`); + res.end('data: [DONE]\n\n'); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ id, object: 'chat.completion', created, model: body.model || model, choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }], usage })); + }); + }); + await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); + const { port: bound } = server.address(); + return { + baseURL: `http://127.0.0.1:${bound}/v1`, + model, + requests, + close: () => new Promise((resolve) => { for (const res of open) res.destroy(); server.close(() => resolve()); }), + }; +}; diff --git a/packages/seats/test/trace-store.test.js b/packages/seats/test/trace-store.test.js new file mode 100644 index 0000000000..8aaa196881 --- /dev/null +++ b/packages/seats/test/trace-store.test.js @@ -0,0 +1,77 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { EVENT_TYPES, traceOf, NULL_TRACE } from '../src/trace/events.js'; +import { RunFailure, runFailure, FAILURE_KINDS } from '../src/failures.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +const withRoot = (fn) => { const root = mkdtempSync(join(tmpdir(), 'seats-trace-')); try { return fn(root); } finally { rmSync(root, { recursive: true, force: true }); } }; + +test('a run appends events in seq order, settles once, and lists newest first', () => withRoot((root) => { + let t = 0; + const store = createTraceStore({ root, now: () => new Date(Date.UTC(2026, 9, 25, 0, 0, t++)) }); + const a = store.startRun({ runId: 'run-aaaaaa', adapter: 'stub' }); + a.emit('run.started', { adapter: 'stub', attempt: 1 }); + a.emit('agent.message_chunk', { text: 'PO' }); + a.emit('agent.message_chunk', { text: 'NG' }); + a.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + assert.equal(a.emit('agent.message_chunk', { text: 'late' }), null, 'nothing is written after settle'); + const b = store.startRun({ runId: 'run-bbbbbb', adapter: 'stub' }); + b.emit('run.started', { adapter: 'stub', attempt: 1 }); + + assert.deepEqual(store.readRun('run-aaaaaa').map((e) => [e.seq, e.type]), [ + [1, 'run.started'], [2, 'agent.message_chunk'], [3, 'agent.message_chunk'], [4, 'run.settled']]); + assert.deepEqual(store.listRuns().map((r) => [r.runId, r.outcome]), [['run-bbbbbb', 'running'], ['run-aaaaaa', 'ok']]); + assert.equal(statSync(join(root, 'runs', 'run-aaaaaa.jsonl')).mode & 0o777, 0o600); + assert.equal(statSync(join(root, 'runs')).mode & 0o777, 0o700); +})); + +test('unknown event types, duplicate and unsafe run ids are refused', () => withRoot((root) => { + const store = createTraceStore({ root }); + const run = store.startRun({ runId: 'run-cccccc' }); + assert.throws(() => run.emit('made.up', {}), /unknown trace event type/); + assert.throws(() => store.startRun({ runId: 'run-cccccc' }), /already exists/); + assert.throws(() => store.startRun({ runId: '../escape' }), /runId/); + assert.equal(store.readRun('../etc/passwd'), null); + assert.ok(EVENT_TYPES.includes('liveness.stalled')); +})); + +test('traceOf falls back to a no-op hook', () => { + assert.equal(traceOf({}), NULL_TRACE); + const hook = { emit() {} }; + assert.equal(traceOf({ trace: hook }), hook); +}); + +test('failures carry a kind, a retry flag, and scrub known credentials', () => { + const error = runFailure('refusal', 'provider said no to sk-or-v1-secretsecretsecret', { credentials: ['sk-or-v1-secretsecretsecret'] }); + assert.ok(error instanceof RunFailure); + assert.equal(error.kind, 'refusal'); + assert.equal(error.retryable, false); + assert.doesNotMatch(error.message, /secretsecret/); + assert.equal(FAILURE_KINDS.stuck.retryable, true); + assert.throws(() => new RunFailure('nope'), /unknown failure kind/); +}); + +test('fake provider answers stream and non-stream turns and honours directives', async () => { + const fake = await startFakeProvider(); + try { + const post = (body) => fetch(`${fake.baseURL}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + const plain = await (await post({ model: 'fake-model', messages: [{ role: 'user', content: 'hi' }] })).json(); + assert.equal(plain.choices[0].message.content, 'PONG'); + const custom = await (await post({ messages: [{ role: 'user', content: 'say [[reply:PONG-7]]' }] })).json(); + assert.equal(custom.choices[0].message.content, 'PONG-7'); + const streamed = await (await post({ stream: true, messages: [{ role: 'user', content: 'hi' }] })).text(); + assert.match(streamed, /"content":"PONG"/); + assert.match(streamed, /data: \[DONE\]/); + assert.equal((await post({ messages: [{ role: 'user', content: '[[refuse]]' }] })).status, 400); + const controller = new AbortController(); + const stalled = post({ messages: [{ role: 'user', content: '[[stall]]' }] }); + setTimeout(() => controller.abort(), 50); + const outcome = await Promise.race([stalled.then(() => 'answered'), new Promise((r) => setTimeout(() => r('held'), 150))]); + assert.equal(outcome, 'held'); + assert.deepEqual(fake.requests.map((r) => r.directive), ['reply', 'reply', 'reply', 'refuse', 'stall']); + } finally { await fake.close(); } +}); From 1c6075753dbf67900149c9aa69db0db4c9fad30e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:43:00 -0700 Subject: [PATCH 03/20] feat(seats): per-run grants for credentials and tool permissions A grant names the credentials one run may hold and sorts its tool calls into allow, ask and deny by ACP kind and title glob (deny > ask > allow, default deny). prepareRunEnv builds the run env from a short allowlist, a throwaway 0700 HOME with XDG dirs inside it, and only the granted credentials, failing credential_denied when one cannot be resolved. createPermissionPolicy turns the grant into ctx.decidePermission: allow picks allow_once, deny picks reject_once, ask waits on the host's approve callback with a timeout, and allow_always is never chosen. Every decision and grant is traced by name, never by value. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/src/index.js | 3 + packages/seats/src/trace/events.js | 4 +- packages/seats/src/valet/env.js | 124 +++++++++++++ packages/seats/src/valet/grant.js | 118 ++++++++++++ packages/seats/src/valet/permissions.js | 90 +++++++++ packages/seats/test/valet-env.test.js | 171 ++++++++++++++++++ packages/seats/test/valet-permissions.test.js | 159 ++++++++++++++++ 7 files changed, 667 insertions(+), 2 deletions(-) create mode 100644 packages/seats/src/valet/env.js create mode 100644 packages/seats/src/valet/grant.js create mode 100644 packages/seats/src/valet/permissions.js create mode 100644 packages/seats/test/valet-env.test.js create mode 100644 packages/seats/test/valet-permissions.test.js diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index 08c968faeb..3ac1f5ce1f 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -7,3 +7,6 @@ export { createHermesAdapter } from './adapters/hermes.js'; export { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; export { EVENT_TYPES, isEventType, traceOf, NULL_TRACE } from './trace/events.js'; export { createTraceStore } from './trace/store.js'; +export { validateGrant, matchToolCall, TOOL_KINDS, CREDENTIAL_SOURCES } from './valet/grant.js'; +export { prepareRunEnv, BASE_ENV_ALLOWLIST } from './valet/env.js'; +export { createPermissionPolicy, DEFAULT_APPROVE_TIMEOUT_MS } from './valet/permissions.js'; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index c78b0633d5..a6a02388a4 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -16,12 +16,12 @@ export const EVENT_TYPES = Object.freeze([ 'tool.update', // { toolCallId, status?, title? } 'plan', // { entries: [{content, status}] } 'permission.requested', // { toolCallId?, options: [{optionId, kind}] } - 'permission.decided', // { outcome: 'selected'|'cancelled', optionId?, kind? } + 'permission.decided', // { outcome: 'selected'|'cancelled', optionId?, kind?, toolCallId?, rule?: {list, pattern}, approval? } 'usage', // { inputTokens?, outputTokens?, totalTokens?, cost? } null-not-zero when unknown 'liveness.stalled', // { idleMs, windowMs } 'liveness.recovery', // { action: 'cancel'|'kill'|'retry', attempt } 'credential.granted', // { names: [...] } names only, never values - 'credential.denied', // { name } + 'credential.denied', // { name, reason?: 'not_granted'|'unresolved' } 'process.exited', // { code, signal } ]); diff --git a/packages/seats/src/valet/env.js b/packages/seats/src/valet/env.js new file mode 100644 index 0000000000..ee38cdf50d --- /dev/null +++ b/packages/seats/src/valet/env.js @@ -0,0 +1,124 @@ +/** + * Build the env and HOME one run executes in. The run starts from nothing the + * user's login can be found through: a short allowlist of locale/terminal + * variables, a fresh HOME whose XDG dirs sit inside it, and only the + * credentials its grant names. + * + * What this is NOT: a filesystem sandbox. A process that already knows an + * absolute path (/Users//.claude) can still open it, and macOS keychain + * items are reached through the user's security session, not through HOME. + * This removes the paths a CLI discovers its login by; an OS sandbox + * (Seatbelt, bwrap) is what stops a process that goes looking. + */ +import { mkdtemp, mkdir, chmod, readFile, rm } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { delimiter, join, resolve, sep } from 'node:path'; +import { traceOf } from '../trace/events.js'; +import { runFailure } from '../failures.js'; +import { validateGrant } from './grant.js'; + +// Same spirit as the goose adapter's SAFE_ENV: locale, terminal and TLS trust +// only. USER/LOGNAME are left out on purpose; they name the account whose +// login this boundary exists to hide. +export const BASE_ENV_ALLOWLIST = /^(?:PATH|LANG|LC_[A-Z0-9_]+|TERM|TZ|SSL_CERT_FILE|SSL_CERT_DIR|NO_COLOR|CI)$/; + +const isInside = (path, dir) => { + const a = resolve(path); + const b = resolve(dir); + return a === b || a.startsWith(b.endsWith(sep) ? b : b + sep); +}; + +/** + * PATH entries under the real home both name it and often hold shims that + * read the user's login (~/.local/bin/claude). Adapters run their CLI by + * absolute path, so the run loses nothing it needs. + */ +const scrubPath = (value, realHomes) => value.split(delimiter) + .filter((entry) => entry && !realHomes.some((home) => isInside(entry, home))) + .join(delimiter); + +const pickBaseEnv = (baseEnv, realHomes) => { + const env = {}; + for (const [name, value] of Object.entries(baseEnv)) { + if (typeof value === 'string' && BASE_ENV_ALLOWLIST.test(name)) env[name] = value; + } + if (env.PATH !== undefined) env.PATH = scrubPath(env.PATH, realHomes); + return env; +}; + +const makeHome = async (root) => { + const home = await mkdtemp(join(root || tmpdir(), 'seat-home-')); + // Conventional layout, so a CLI that ignores XDG_* and defaults to + // ~/.config still lands inside the throwaway HOME. + const dirs = { + XDG_CONFIG_HOME: join(home, '.config'), + XDG_DATA_HOME: join(home, '.local', 'share'), + XDG_STATE_HOME: join(home, '.local', 'state'), + XDG_CACHE_HOME: join(home, '.cache'), + TMPDIR: join(home, 'tmp'), + }; + await chmod(home, 0o700); + for (const dir of Object.values(dirs)) await mkdir(dir, { recursive: true, mode: 0o700 }); + // mkdir's mode is masked by umask; set it explicitly on every level made. + for (const dir of [join(home, '.local'), ...Object.values(dirs)]) await chmod(dir, 0o700); + return { home, dirs }; +}; + +/** A value that cannot ride an env var is the same as no value. */ +const usable = (value) => typeof value === 'string' && value.length > 0 && !value.includes('\0'); + +const resolveOne = async (spec, { baseEnv, resolveSecret }) => { + if (spec.from === 'env') return baseEnv[spec.ref || spec.name]; + if (spec.from === 'file') return (await readFile(spec.ref, 'utf8')).replace(/\r?\n$/, ''); + if (typeof resolveSecret !== 'function') return undefined; + return resolveSecret(spec.name, spec); +}; + +/** + * prepareRunEnv({ grant, baseEnv, resolveSecret, root?, require?, trace?, ctx? }) + * → { env, home, cleanup, granted: [names] } + * + * `require` lists credential names the adapter cannot run without; one the + * grant does not name fails the run as credential_denied before any spawn. + * A named credential that cannot be resolved fails it the same way: a run + * that silently lacks its key fails later and less legibly. + * + * Events go to `trace`, else ctx.trace: credential.granted {names} and + * credential.denied {name, reason}. Values never enter an event or a message. + */ +export const prepareRunEnv = async ({ + grant, baseEnv = {}, resolveSecret, root, require: required = [], trace, ctx, +} = {}) => { + const valid = validateGrant(grant); + const hook = trace && typeof trace.emit === 'function' ? trace : traceOf(ctx); + const realHomes = [...new Set([baseEnv.HOME, homedir()].filter(Boolean))]; + const deny = (name, reason, values) => { + hook.emit('credential.denied', { name, reason }); + return runFailure('credential_denied', `${name} ${reason === 'not_granted' ? 'is not in the run grant' : 'could not be resolved'}`, { credentials: values }); + }; + + const granted = new Set(valid.credentials.map((c) => c.name)); + const missing = required.find((name) => !granted.has(name)); + if (missing) throw deny(missing, 'not_granted', []); + + const { home, dirs } = await makeHome(root); + const cleanup = () => rm(home, { recursive: true, force: true }); + const env = { ...pickBaseEnv(baseEnv, realHomes), HOME: home, ...dirs }; + const values = []; + try { + for (const spec of valid.credentials) { + let value; + // A resolver's error text may quote the secret it failed on; drop it. + try { value = await resolveOne(spec, { baseEnv, resolveSecret }); } catch { value = undefined; } + if (!usable(value)) throw deny(spec.name, 'unresolved', values); + values.push(value); + env[spec.name] = value; + } + } catch (error) { + await cleanup(); + throw error; + } + const names = valid.credentials.map((c) => c.name); + hook.emit('credential.granted', { names }); + return { env, home, cleanup, granted: names }; +}; diff --git a/packages/seats/src/valet/grant.js b/packages/seats/src/valet/grant.js new file mode 100644 index 0000000000..b2109bfd39 --- /dev/null +++ b/packages/seats/src/valet/grant.js @@ -0,0 +1,118 @@ +/** + * A grant says what one run may hold and do: which credentials reach its env, + * and which tool calls it may make without asking, only after asking, or never. + * + * { + * credentials: [{ name: 'OPENAI_API_KEY', from: 'env'|'file'|'resolver', ref? }], + * tools: { allow: [pattern], ask: [pattern], deny: [pattern] }, + * } + * + * `from: 'env'` copies baseEnv[ref ?? name]; `from: 'file'` reads the absolute + * path in `ref`; `from: 'resolver'` asks the host's resolveSecret(name, spec), + * with `ref` as an opaque hint for it. + * + * A tool pattern is `` or `:`, where kind is one of + * ACP's ToolKind values or `*`. `execute` matches every execute call; + * `execute:git status*` matches execute calls whose title starts that way; + * `*:*.env*` matches any kind by title. Globs know `*` and `?` only. + * + * The title is text the agent chose. A title-glob deny narrows what the agent + * can do; a title-glob allow is only as strong as the agent's honesty about + * its title, so prefer kind-level allows and put execute behind ask. + * + * Precedence is deny > ask > allow, and a call no rule names is denied. + * + * `network` rules are refused for now rather than accepted and ignored: a run + * would believe it was fenced when nothing enforces the fence. TODO: egress + * allowlist once a sandbox can enforce it. + */ + +export const TOOL_KINDS = Object.freeze(['read', 'edit', 'delete', 'move', 'search', 'execute', 'think', 'fetch', 'other']); +export const CREDENTIAL_SOURCES = Object.freeze(['env', 'file', 'resolver']); +const RULE_LISTS = ['deny', 'ask', 'allow']; // checked in this order: the precedence +const VALIDATED = Symbol('validated grant'); + +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/; +// Names the run env is built from. A credential under one of these names would +// overwrite the boundary itself (a granted HOME points back at the real login). +const RESERVED_NAME = /^(?:HOME|PATH|TMPDIR|USER|LOGNAME|SHELL|PWD|LANG|LC_[A-Z0-9_]+|TERM|TZ|NO_COLOR|CI|SSL_CERT_FILE|SSL_CERT_DIR|XDG_[A-Z0-9_]+)$/; + +const fail = (message) => { throw new Error(`grant: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); + } +}; + +const globToRegExp = (glob) => new RegExp(`^${glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[\\s\\S]*').replace(/\?/g, '[\\s\\S]')}$`); + +/** Parse once at validation so matching never re-reads a malformed pattern. */ +const parsePattern = (pattern, where) => { + if (typeof pattern !== 'string' || !pattern.trim()) fail(`${where} must be a non-empty string`); + const colon = pattern.indexOf(':'); + const kind = colon === -1 ? pattern : pattern.slice(0, colon); + const title = colon === -1 ? null : pattern.slice(colon + 1); + if (kind !== '*' && !TOOL_KINDS.includes(kind)) fail(`${where} names unknown tool kind "${kind}"`); + if (title !== null && !title) fail(`${where} has an empty title glob`); + return Object.freeze({ pattern, kind, title: title === null ? null : globToRegExp(title) }); +}; + +const validateCredential = (credential, index, seen) => { + const where = `credentials[${index}]`; + if (!isPlainObject(credential)) fail(`${where} must be an object`); + refuseUnknownKeys(credential, ['name', 'from', 'ref'], `${where}.`); + const { name, from, ref } = credential; + if (typeof name !== 'string' || !ENV_NAME.test(name)) fail(`${where}.name must be an ENV_NAME (A-Z, 0-9, _)`); + if (RESERVED_NAME.test(name)) fail(`${where}.name ${name} is reserved for the run env`); + if (seen.has(name)) fail(`${where}.name ${name} is granted twice`); + seen.add(name); + if (!CREDENTIAL_SOURCES.includes(from)) fail(`${where}.from must be one of ${CREDENTIAL_SOURCES.join(', ')}`); + if (ref !== undefined && (typeof ref !== 'string' || !ref)) fail(`${where}.ref must be a non-empty string`); + if (from === 'env' && ref !== undefined && !ENV_NAME.test(ref)) fail(`${where}.ref must be an ENV_NAME for from: env`); + if (from === 'file' && (ref === undefined || !ref.startsWith('/'))) fail(`${where}.ref must be an absolute path for from: file`); + return Object.freeze(ref === undefined ? { name, from } : { name, from, ref }); +}; + +/** Validate and freeze a grant. Throws on the first problem; never repairs one. */ +export const validateGrant = (grant) => { + if (grant && grant[VALIDATED]) return grant; + if (!isPlainObject(grant)) fail('must be an object'); + if ('network' in grant) fail('network rules are not enforced yet; remove the field rather than rely on it'); + refuseUnknownKeys(grant, ['credentials', 'tools'], ''); + const credentials = grant.credentials ?? []; + if (!Array.isArray(credentials)) fail('credentials must be an array'); + const seen = new Set(); + const tools = grant.tools ?? {}; + if (!isPlainObject(tools)) fail('tools must be an object'); + refuseUnknownKeys(tools, RULE_LISTS, 'tools.'); + const rules = {}; + for (const list of RULE_LISTS) { + const patterns = tools[list] ?? []; + if (!Array.isArray(patterns)) fail(`tools.${list} must be an array`); + rules[list] = Object.freeze(patterns.map((pattern, i) => parsePattern(pattern, `tools.${list}[${i}]`))); + } + return Object.freeze({ + [VALIDATED]: true, + credentials: Object.freeze(credentials.map((c, i) => validateCredential(c, i, seen))), + tools: Object.freeze(rules), + }); +}; +const matches = (parsed, kind, title) => (parsed.kind === '*' || parsed.kind === kind) + && (parsed.title === null || parsed.title.test(title)); + +/** + * Which rule decides this tool call. ACP makes `kind` optional and means + * `other` when it is absent, so that is what an unknown or missing kind is. + * Returns { decision: 'allow'|'ask'|'deny', rule: { list, pattern } }. + */ +export const matchToolCall = (grant, toolCall = {}) => { + const valid = validateGrant(grant); + const kind = TOOL_KINDS.includes(toolCall?.kind) ? toolCall.kind : 'other'; + const title = typeof toolCall?.title === 'string' ? toolCall.title : ''; + for (const list of RULE_LISTS) { + const hit = valid.tools[list].find((parsed) => matches(parsed, kind, title)); + if (hit) return { decision: list, rule: { list, pattern: hit.pattern } }; + } + return { decision: 'deny', rule: { list: 'default', pattern: null } }; +}; diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js new file mode 100644 index 0000000000..6a2f609ad8 --- /dev/null +++ b/packages/seats/src/valet/permissions.js @@ -0,0 +1,90 @@ +/** + * Turn a grant into ctx.decidePermission: the answer to ACP's + * session/request_permission, decided per tool call. + * + * allow → the allow_once option + * deny → the reject_once option, or cancelled when there is none + * ask → approve({ toolCall, grantRule }) from the host; true allows, + * anything else (false, a throw, no answer in time) denies + * + * allow_always and reject_always are never chosen: either would be remembered + * by the agent past this grant, so the next call would skip the check. + * An allow rule with no allow_once option therefore denies. + */ +import { traceOf } from '../trace/events.js'; +import { matchToolCall, validateGrant } from './grant.js'; + +export const DEFAULT_APPROVE_TIMEOUT_MS = 5 * 60 * 1000; + +const optionOfKind = (options, kind) => (Array.isArray(options) ? options : []) + .find((option) => option && option.kind === kind && typeof option.optionId === 'string'); + +const allowAnswer = (options) => { + const option = optionOfKind(options, 'allow_once'); + return option ? { outcome: 'selected', optionId: option.optionId, kind: option.kind } : null; +}; +const denyAnswer = (options) => { + const option = optionOfKind(options, 'reject_once'); + return option ? { outcome: 'selected', optionId: option.optionId, kind: option.kind } : { outcome: 'cancelled' }; +}; + +/** + * Wait for the host's approval, bounded. Resolves 'approved' | 'rejected' | + * 'timed_out' | 'failed' | 'aborted'; never rejects, so a broken approver + * becomes a denial rather than a hung or crashed turn. + */ +const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((done) => { + if (typeof approve !== 'function') { done('failed'); return; } + if (signal?.aborted) { done('aborted'); return; } + let finished = false; + const finish = (verdict) => { + if (finished) return; + finished = true; + clearTimeout(timer); + signal?.removeEventListener?.('abort', onAbort); + done(verdict); + }; + const onAbort = () => finish('aborted'); + const timer = setTimeout(() => finish('timed_out'), timeoutMs); + signal?.addEventListener?.('abort', onAbort, { once: true }); + Promise.resolve() + .then(() => approve(input)) + .then((answer) => finish(answer === true ? 'approved' : 'rejected'), () => finish('failed')); +}); + +/** + * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal? }) + * → (request) => Promise<{outcome:'selected', optionId} | {outcome:'cancelled'}> + * + * `signal` is the run's AbortSignal: a cancelled run answers a pending ask + * with cancelled, which is what ACP asks of a client after session/cancel. + */ +export const createPermissionPolicy = (grant, { + approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, +} = {}) => { + const valid = validateGrant(grant); + const hook = traceOf({ trace }); + return async (request = {}) => { + const toolCall = request?.toolCall || {}; + const options = request?.options; + const { decision, rule } = matchToolCall(valid, toolCall); + let approval; + let answer; + if (decision === 'ask') { + approval = await awaitApproval(approve, { toolCall, grantRule: rule }, { timeoutMs: approveTimeoutMs, signal }); + if (approval === 'aborted') answer = { outcome: 'cancelled' }; + else answer = approval === 'approved' ? allowAnswer(options) : null; + } else if (decision === 'allow') { + answer = allowAnswer(options); + } + if (!answer) answer = denyAnswer(options); + hook.emit('permission.decided', { + outcome: answer.outcome, + ...(answer.optionId ? { optionId: answer.optionId, kind: answer.kind } : {}), + toolCallId: typeof toolCall.toolCallId === 'string' ? toolCall.toolCallId : null, + rule, + ...(approval ? { approval } : {}), + }); + return answer.outcome === 'selected' ? { outcome: 'selected', optionId: answer.optionId } : { outcome: 'cancelled' }; + }; +}; diff --git a/packages/seats/test/valet-env.test.js b/packages/seats/test/valet-env.test.js new file mode 100644 index 0000000000..5a35adf191 --- /dev/null +++ b/packages/seats/test/valet-env.test.js @@ -0,0 +1,171 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, readdirSync as readdir, rmSync, statSync, writeFileSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { RunFailure } from '../src/failures.js'; + +const REAL_HOME = homedir(); +const GRANTED = 'sk-granted-0123456789abcdef'; +const WITHHELD = 'sk-withheld-fedcba9876543210'; + +const recorder = () => { const events = []; return { events, emit: (type, data) => events.push({ type, data }) }; }; +const withScratch = async (fn) => { + const dir = mkdtempSync(join(tmpdir(), 'seats-valet-')); + try { return await fn(dir); } finally { rmSync(dir, { recursive: true, force: true }); } +}; + +// The child reports what it can see; the assertions run in the parent. +const PROBE = ` +const os = require('os'); const fs = require('fs'); const path = require('path'); +process.stdout.write(JSON.stringify({ + env: process.env, + homedir: os.homedir(), + claude: fs.existsSync(path.join(os.homedir(), '.claude')), + codex: fs.existsSync(path.join(os.homedir(), '.codex')), +}));`; +const probe = (env, cwd) => { + const child = spawnSync(process.execPath, ['-e', PROBE], { env, cwd, encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + return JSON.parse(child.stdout); +}; + +test('a real child sees only the granted key, a throwaway HOME, and no path to the real login', () => withScratch(async (root) => { + // The parent's own env, with both secrets planted, is the base the run is cut from. + const baseEnv = { ...process.env, HOME: REAL_HOME, OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD, CLAUDE_CONFIG_DIR: join(REAL_HOME, '.claude') }; + const trace = recorder(); + const run = await prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }] }, baseEnv, root, trace, + }); + try { + const seen = probe(run.env, run.home); + assert.equal(seen.env.OPENAI_API_KEY, GRANTED, 'the granted key reaches the child'); + assert.equal(seen.env.ANTHROPIC_API_KEY, undefined, 'a parent secret the grant does not name is absent'); + assert.equal(seen.env.CLAUDE_CONFIG_DIR, undefined); + assert.ok(!JSON.stringify(seen.env).includes(WITHHELD)); + assert.equal(seen.env.HOME, run.home); + assert.equal(seen.homedir, run.home); + assert.ok(run.home.startsWith(root)); + for (const [name, value] of Object.entries(seen.env)) { + assert.ok(!value.includes(REAL_HOME), `${name} leaks the real HOME path: ${value}`); + } + // Positive control: the real login dir exists, so "not reachable" is a real result. + if (existsSync(join(REAL_HOME, '.claude'))) assert.equal(seen.claude, false, '~/.claude is not reachable via HOME'); + else assert.fail('positive control missing: no ~/.claude on this host'); + if (existsSync(join(REAL_HOME, '.codex'))) assert.equal(seen.codex, false, '~/.codex is not reachable via HOME'); + else assert.fail('positive control missing: no ~/.codex on this host'); + for (const name of ['XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME', 'XDG_CACHE_HOME', 'TMPDIR']) { + assert.ok(seen.env[name].startsWith(`${run.home}/`), `${name} sits inside the run HOME`); + assert.equal(statSync(seen.env[name]).mode & 0o777, 0o700, `${name} is 0700`); + } + assert.equal(statSync(run.home).mode & 0o777, 0o700); + assert.deepEqual(run.granted, ['OPENAI_API_KEY']); + assert.deepEqual(trace.events, [{ type: 'credential.granted', data: { names: ['OPENAI_API_KEY'] } }]); + } finally { + await run.cleanup(); + } + assert.equal(existsSync(run.home), false, 'cleanup removes the HOME'); +})); + +test('the base env keeps only the allowlist, and PATH loses entries under the real home', () => withScratch(async (root) => { + const baseEnv = { + HOME: REAL_HOME, PATH: `/usr/bin:${REAL_HOME}/.local/bin:/bin`, LANG: 'en_US.UTF-8', LC_ALL: 'C', TERM: 'xterm', TZ: 'UTC', + SSL_CERT_FILE: '/etc/ssl/cert.pem', NO_COLOR: '1', CI: '1', USER: 'someone', SSH_AUTH_SOCK: '/tmp/agent.sock', AWS_PROFILE: 'prod', GH_TOKEN: 'ghp_x', + }; + const run = await prepareRunEnv({ grant: {}, baseEnv, root }); + try { + assert.deepEqual(Object.keys(run.env).sort(), [ + 'CI', 'HOME', 'LANG', 'LC_ALL', 'NO_COLOR', 'PATH', 'SSL_CERT_FILE', 'TERM', 'TMPDIR', 'TZ', + 'XDG_CACHE_HOME', 'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME']); + assert.equal(run.env.PATH, '/usr/bin:/bin'); + } finally { await run.cleanup(); } +})); + +test('file and resolver credentials resolve; resolveSecret sees the name and spec', () => withScratch(async (root) => { + const keyFile = join(root, 'key.txt'); + writeFileSync(keyFile, 'from-file-value\n', { mode: 0o600 }); + const calls = []; + const run = await prepareRunEnv({ + grant: { credentials: [ + { name: 'FILE_KEY', from: 'file', ref: keyFile }, + { name: 'VAULT_KEY', from: 'resolver', ref: 'vault://team/key' }, + { name: 'RENAMED_KEY', from: 'env', ref: 'SOURCE_KEY' }, + ] }, + baseEnv: { SOURCE_KEY: 'from-env-value' }, + resolveSecret: async (name, spec) => { calls.push([name, spec.ref]); return 'from-resolver-value'; }, + root, + }); + try { + assert.equal(run.env.FILE_KEY, 'from-file-value'); + assert.equal(run.env.VAULT_KEY, 'from-resolver-value'); + assert.equal(run.env.RENAMED_KEY, 'from-env-value'); + assert.equal(run.env.SOURCE_KEY, undefined, 'the source name is not copied across too'); + assert.deepEqual(calls, [['VAULT_KEY', 'vault://team/key']]); + } finally { await run.cleanup(); } +})); + +test('an unresolvable credential fails credential_denied, emits its name only, and leaves no HOME', () => withScratch(async (root) => { + const cases = [ + { name: 'resolver returns nothing', resolveSecret: async () => undefined }, + { name: 'resolver throws with the value in its message', resolveSecret: async () => { throw new Error(`boom ${GRANTED}`); } }, + { name: 'resolver returns empty', resolveSecret: async () => '' }, + { name: 'no resolver at all', resolveSecret: undefined }, + ]; + for (const c of cases) { + const trace = recorder(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }, { name: 'VAULT_KEY', from: 'resolver' }] }, + baseEnv: { OPENAI_API_KEY: GRANTED }, resolveSecret: c.resolveSecret, root, trace, + }), (error) => { + assert.ok(error instanceof RunFailure, c.name); + assert.equal(error.kind, 'credential_denied'); + assert.match(error.message, /VAULT_KEY/); + assert.ok(!error.message.includes(GRANTED)); + return true; + }); + assert.deepEqual(trace.events, [{ type: 'credential.denied', data: { name: 'VAULT_KEY', reason: 'unresolved' } }], c.name); + } + await assert.rejects(prepareRunEnv({ grant: { credentials: [{ name: 'GONE', from: 'file', ref: join(root, 'missing') }] }, root }), /credential_denied/); + await assert.rejects(prepareRunEnv({ grant: { credentials: [{ name: 'GONE', from: 'env' }] }, baseEnv: {}, root }), /credential_denied/); + assert.deepEqual(readdir(root), [], 'every failed prepare removed its HOME'); +})); + +test('a credential the adapter requires but the grant does not name is denied before any HOME is made', () => withScratch(async (root) => { + const trace = recorder(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [] }, baseEnv: { ANTHROPIC_API_KEY: WITHHELD }, require: ['ANTHROPIC_API_KEY'], root, trace, + }), (error) => error.kind === 'credential_denied' && /not in the run grant/.test(error.message)); + assert.deepEqual(trace.events, [{ type: 'credential.denied', data: { name: 'ANTHROPIC_API_KEY', reason: 'not_granted' } }]); + assert.deepEqual(readdir(root), []); +})); + +test('no credential value ever reaches the serialized trace store', () => withScratch(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'valet-scan-1' }); + const keyFile = join(root, 'k'); + writeFileSync(keyFile, 'sk-file-secret-aaaaaaaaaaaa'); + const prepared = await prepareRunEnv({ + grant: { credentials: [ + { name: 'OPENAI_API_KEY', from: 'env' }, + { name: 'FILE_KEY', from: 'file', ref: keyFile }, + { name: 'VAULT_KEY', from: 'resolver' }, + ] }, + baseEnv: { OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD }, + resolveSecret: async () => 'sk-resolver-secret-bbbbbbbbbbbb', + root, + ctx: { trace: run }, + }); + await prepared.cleanup(); + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }, { name: 'VAULT_KEY', from: 'resolver' }] }, + baseEnv: { OPENAI_API_KEY: GRANTED }, resolveSecret: async () => { throw new Error(GRANTED); }, root, ctx: { trace: run }, + })); + const raw = readFileSync(join(root, 'trace', 'runs', 'valet-scan-1.jsonl'), 'utf8') + readFileSync(join(root, 'trace', 'index.jsonl'), 'utf8'); + assert.deepEqual(store.readRun('valet-scan-1').map((e) => e.type), ['credential.granted', 'credential.denied']); + for (const value of [GRANTED, WITHHELD, 'sk-file-secret-aaaaaaaaaaaa', 'sk-resolver-secret-bbbbbbbbbbbb']) { + assert.ok(!raw.includes(value), `trace carries a credential value: ${value}`); + } +})); diff --git a/packages/seats/test/valet-permissions.test.js b/packages/seats/test/valet-permissions.test.js new file mode 100644 index 0000000000..c528c0f8e5 --- /dev/null +++ b/packages/seats/test/valet-permissions.test.js @@ -0,0 +1,159 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { validateGrant, matchToolCall } from '../src/valet/grant.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import * as seats from '../src/index.js'; + +const OPTIONS = [ + { optionId: 'opt-always', name: 'Always allow', kind: 'allow_always' }, + { optionId: 'opt-once', name: 'Allow', kind: 'allow_once' }, + { optionId: 'opt-reject', name: 'Reject', kind: 'reject_once' }, + { optionId: 'opt-reject-always', name: 'Never', kind: 'reject_always' }, +]; +const request = (kind, title = 'a tool', options = OPTIONS) => ({ + sessionId: 's-1', toolCall: { toolCallId: `tc-${kind}`, title, kind }, options, +}); +const recorder = () => { const events = []; return { events, emit: (type, data) => events.push({ type, data }) }; }; + +test('precedence is deny > ask > allow > default deny', () => { + const grant = validateGrant({ tools: { + allow: ['read', 'search', 'execute:git *', '*:open *'], + ask: ['execute', 'edit:*.md'], + deny: ['execute:git push*', 'read:*.env*', 'edit:secrets/*'], + } }); + const table = [ + // [kind, title, decision, list, pattern] + ['read', 'README.md', 'allow', 'allow', 'read'], + ['read', 'config/.env.local', 'deny', 'deny', 'read:*.env*'], + ['search', 'grep TODO', 'allow', 'allow', 'search'], + ['execute', 'git status', 'ask', 'ask', 'execute'], + ['execute', 'git push origin main', 'deny', 'deny', 'execute:git push*'], + ['execute', 'npm test', 'ask', 'ask', 'execute'], + ['edit', 'notes.md', 'ask', 'ask', 'edit:*.md'], + ['edit', 'secrets/notes.md', 'deny', 'deny', 'edit:secrets/*'], + ['edit', 'src/a.js', 'deny', 'default', null], + ['fetch', 'open https://x', 'allow', 'allow', '*:open *'], + ['delete', 'rm -rf /', 'deny', 'default', null], + [undefined, 'mystery', 'deny', 'default', null], + ['not-a-kind', 'open thing', 'allow', 'allow', '*:open *'], + ]; + for (const [kind, title, decision, list, pattern] of table) { + assert.deepEqual(matchToolCall(grant, { kind, title }), { decision, rule: { list, pattern } }, `${kind} ${title}`); + } + // A missing kind means ACP's `other`, and `other` can be granted like any kind. + assert.equal(matchToolCall({ tools: { allow: ['other'] } }, { title: 'x' }).decision, 'allow'); + assert.equal(matchToolCall({}, { kind: 'read', title: 'x' }).decision, 'deny', 'an empty grant denies everything'); +}); + +test('globs are anchored and treat regex characters literally', () => { + const grant = { tools: { allow: ['execute:ls ?', 'read:a.b(c)'] } }; + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'ls a' }).decision, 'allow'); + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'ls ab' }).decision, 'deny'); + assert.equal(matchToolCall(grant, { kind: 'execute', title: 'xls a' }).decision, 'deny'); + assert.equal(matchToolCall(grant, { kind: 'read', title: 'a.b(c)' }).decision, 'allow'); + assert.equal(matchToolCall(grant, { kind: 'read', title: 'aXb(c)' }).decision, 'deny'); +}); + +test('grant validation refuses unknown fields, bad names and bad patterns', () => { + const bad = [ + [null, /must be an object/], + [{ credential: [] }, /unknown field credential/], + [{ network: { allow: ['api.x'] } }, /network rules are not enforced yet/], + [{ tools: { allow: [], permit: [] } }, /unknown field tools\.permit/], + [{ tools: { allow: 'read' } }, /tools\.allow must be an array/], + [{ tools: { allow: ['write'] } }, /unknown tool kind "write"/], + [{ tools: { deny: ['execute:'] } }, /empty title glob/], + [{ tools: { ask: [''] } }, /non-empty string/], + [{ credentials: [{ name: 'openai_key', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: '1KEY', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: 'KEY-X', from: 'env' }] }, /ENV_NAME/], + [{ credentials: [{ name: 'HOME', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'PATH', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'XDG_CONFIG_HOME', from: 'env' }] }, /reserved/], + [{ credentials: [{ name: 'KEY', from: 'keychain' }] }, /from must be one of/], + [{ credentials: [{ name: 'KEY', from: 'env', value: 'sk-x' }] }, /unknown field credentials\[0\]\.value/], + [{ credentials: [{ name: 'KEY', from: 'file' }] }, /absolute path/], + [{ credentials: [{ name: 'KEY', from: 'file', ref: 'rel/key' }] }, /absolute path/], + [{ credentials: [{ name: 'KEY', from: 'env', ref: 'lower' }] }, /ENV_NAME for from: env/], + [{ credentials: [{ name: 'KEY', from: 'env' }, { name: 'KEY', from: 'resolver' }] }, /granted twice/], + ]; + for (const [grant, message] of bad) assert.throws(() => validateGrant(grant), message, JSON.stringify(grant)); + const ok = validateGrant({ credentials: [{ name: 'OPENAI_API_KEY', from: 'resolver', ref: 'vault://k' }], tools: { allow: ['read'] } }); + assert.ok(Object.isFrozen(ok) && Object.isFrozen(ok.credentials[0])); + assert.equal(validateGrant(ok), ok, 'a validated grant passes through'); +}); + +test('allow selects allow_once and never allow_always', async () => { + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { allow: ['read'] } }, { trace }); + assert.deepEqual(await decide(request('read')), { outcome: 'selected', optionId: 'opt-once' }); + // Only allow_always offered: an allow rule still must not pick it. + const onlyAlways = [OPTIONS[0], OPTIONS[2]]; + assert.deepEqual(await decide(request('read', 'x', onlyAlways)), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(await decide(request('read', 'x', [OPTIONS[0]])), { outcome: 'cancelled' }); + assert.deepEqual(trace.events[0], { type: 'permission.decided', data: { + outcome: 'selected', optionId: 'opt-once', kind: 'allow_once', toolCallId: 'tc-read', rule: { list: 'allow', pattern: 'read' }, + } }); +}); + +test('deny selects reject_once, or cancelled when there is no reject_once option', async () => { + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { deny: ['execute'] } }, { trace }); + assert.deepEqual(await decide(request('execute')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(await decide(request('execute', 'x', [OPTIONS[0], OPTIONS[1], OPTIONS[3]])), { outcome: 'cancelled' }); + assert.deepEqual(await decide(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }, 'default deny'); + assert.deepEqual(await decide({}), { outcome: 'cancelled' }, 'a malformed request is denied, not thrown'); + assert.deepEqual(trace.events.map((e) => [e.data.outcome, e.data.rule.list]), [ + ['selected', 'deny'], ['cancelled', 'deny'], ['selected', 'default'], ['cancelled', 'default']]); +}); + +test('ask: approved allows once, rejected denies, and approve sees the tool call and rule', async () => { + const seen = []; + const trace = recorder(); + let verdict = true; + const decide = createPermissionPolicy({ tools: { ask: ['execute'] } }, { + trace, approve: async (input) => { seen.push(input); return verdict; }, + }); + const req = request('execute', 'npm test'); + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-once' }); + verdict = false; + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }); + verdict = 'yes'; + assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }, 'only literal true approves'); + assert.deepEqual(seen[0], { toolCall: req.toolCall, grantRule: { list: 'ask', pattern: 'execute' } }); + assert.deepEqual(trace.events.map((e) => [e.data.kind, e.data.approval]), [ + ['allow_once', 'approved'], ['reject_once', 'rejected'], ['reject_once', 'rejected']]); +}); + +test('ask: no answer in time denies; a throwing or missing approver denies', async () => { + const trace = recorder(); + const started = Date.now(); + const slow = createPermissionPolicy({ tools: { ask: ['edit'] } }, { + trace, approveTimeoutMs: 30, approve: () => new Promise(() => {}), + }); + assert.deepEqual(await slow(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.ok(Date.now() - started < 1000); + const throwing = createPermissionPolicy({ tools: { ask: ['edit'] } }, { trace, approve: () => { throw new Error('card service down'); } }); + assert.deepEqual(await throwing(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + const none = createPermissionPolicy({ tools: { ask: ['edit'] } }, { trace }); + assert.deepEqual(await none(request('edit')), { outcome: 'selected', optionId: 'opt-reject' }); + assert.deepEqual(trace.events.map((e) => e.data.approval), ['timed_out', 'failed', 'failed']); +}); + +test('ask: a cancelled run answers a pending ask with cancelled', async () => { + const controller = new AbortController(); + const trace = recorder(); + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace, signal: controller.signal, approveTimeoutMs: 10_000, approve: () => new Promise(() => {}), + }); + const pending = decide(request('fetch')); + setTimeout(() => controller.abort(), 5); + assert.deepEqual(await pending, { outcome: 'cancelled' }); + assert.equal(trace.events[0].data.approval, 'aborted'); +}); + +test('the valet API is exported from the package entry', () => { + for (const name of ['validateGrant', 'matchToolCall', 'prepareRunEnv', 'createPermissionPolicy', 'TOOL_KINDS', 'BASE_ENV_ALLOWLIST']) { + assert.ok(name in seats, name); + } +}); From 099db50fb1f86b9d96b1359cb33b282b0b7373ab Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:20:23 -0700 Subject: [PATCH 04/20] feat(seats): run/grant record format v1, hash-chained and exportable to OTLP The trace store now writes each run as a versioned record: one canonical (RFC 8785) JSON line per entry, each chained to the previous line by SHA-256. A tool outside this package can read it, verify it and export it. It records who granted which agent what access for which board task, who decided each call and in what role, what was attempted, and the result. - src/record/format.js: appendEntry, verifyRecord (reports torn, seq_mismatch, prev_mismatch, not_canonical, ... at the seq observed), the kinds and their validators. podId is required with taskId, grant scope has the room-grant shape (exact tool names, connectionId, writeMode, budget, audience, parentGrantId), grant.revoked is {grantIds, reason}, task.done is a board-written kind, and actor.id is always an id. - src/record/otel.js: toOtlpJson with deterministic ids, mapped onto the GenAI conventions (Development status upstream, pinned in the doc). - trace store: startRun takes taskId/podId/actor; readRun keeps `type` and adds the envelope; verifyRun adds an index head check. task.done ends task-bound grants; revokeGrant cascades along a lineage; known credential values are scrubbed before hashing. - valet: grant.issued from grantMeta, and permission.decided carries decidedBy/decidedByRole/decisionCardId. Only approved === true approves. - docs/record-v1.md, plus docs/examples generated by a drift test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/README.md | 2 + .../seats/docs/examples/TASK-101.otlp.json | 2285 +++++++++++++++++ .../seats/docs/examples/run-ana-0001.jsonl | 29 + .../seats/docs/examples/run-ben-0001.jsonl | 12 + packages/seats/docs/record-v1.md | 431 ++++ packages/seats/src/index.js | 4 + packages/seats/src/record/format.js | 451 ++++ packages/seats/src/record/otel.js | 214 ++ packages/seats/src/trace/events.js | 14 +- packages/seats/src/trace/store.js | 166 +- packages/seats/src/valet/env.js | 41 +- packages/seats/src/valet/permissions.js | 74 +- packages/seats/test-support/record-sample.js | 142 + packages/seats/test/record-example.test.js | 71 + packages/seats/test/record-format.test.js | 254 ++ packages/seats/test/record-otel.test.js | 131 + packages/seats/test/record-store.test.js | 151 ++ packages/seats/test/record-valet.test.js | 145 ++ 18 files changed, 4578 insertions(+), 39 deletions(-) create mode 100644 packages/seats/docs/examples/TASK-101.otlp.json create mode 100644 packages/seats/docs/examples/run-ana-0001.jsonl create mode 100644 packages/seats/docs/examples/run-ben-0001.jsonl create mode 100644 packages/seats/docs/record-v1.md create mode 100644 packages/seats/src/record/format.js create mode 100644 packages/seats/src/record/otel.js create mode 100644 packages/seats/test-support/record-sample.js create mode 100644 packages/seats/test/record-example.test.js create mode 100644 packages/seats/test/record-format.test.js create mode 100644 packages/seats/test/record-otel.test.js create mode 100644 packages/seats/test/record-store.test.js create mode 100644 packages/seats/test/record-valet.test.js diff --git a/packages/seats/README.md b/packages/seats/README.md index 04e57fd9af..8f86eff12d 100644 --- a/packages/seats/README.md +++ b/packages/seats/README.md @@ -7,3 +7,5 @@ It is private and is not published on its own. `@commonlyai/cli` depends on it a **Rule:** nothing here imports Commonly code. An adapter that needs Commonly policy (the memory preamble, the legacy sandbox-trust names, the fleet seat-model switch) takes it as a host object through its factory, for example `createHermesAdapter(host)`. The CLI binds that host in `cli/src/lib/adapters/<name>.js`. `cli/__tests__/seats-package.test.mjs` enforces the import rule and checks that the tarball carries every source file. The extraction plan is D62, extract-in-place. + +**Run record.** The trace store writes each run as a versioned, hash-chained JSONL record. It covers who granted which agent what access for which task, who approved each call, what was attempted, and the result. A tool outside this package can read, verify and export it to OTLP. The format is specified in [`docs/record-v1.md`](docs/record-v1.md), and a generated sample is in [`docs/examples/`](docs/examples/). diff --git a/packages/seats/docs/examples/TASK-101.otlp.json b/packages/seats/docs/examples/TASK-101.otlp.json new file mode 100644 index 0000000000..8d26baa11f --- /dev/null +++ b/packages/seats/docs/examples/TASK-101.otlp.json @@ -0,0 +1,2285 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "seats-sample" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "seats.record", + "version": "1" + }, + "spans": [ + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "4f7ea2299f50ef90", + "name": "seats.run", + "kind": 1, + "startTimeUnixNano": "1792944002000000000", + "endTimeUnixNano": "1792944030000000000", + "attributes": [ + { + "key": "seats.run.id", + "value": { + "stringValue": "run-ana-0001" + } + }, + { + "key": "seats.task.id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.pod.id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.run.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "seats.run.attempts", + "value": { + "intValue": "2" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944002000000000", + "name": "task.claimed", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.lease_until", + "value": { + "stringValue": "2026-10-25T18:00:00Z" + } + } + ] + }, + { + "timeUnixNano": "1792944003000000000", + "name": "grant.issued", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.binding.ends_on", + "value": { + "stringValue": "task.done" + } + }, + { + "key": "seats.binding.pod_id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.binding.task_id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.expires_at", + "value": { + "stringValue": "2026-10-25T20:00:00.000Z" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.issued_by", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.parent_grant_id", + "value": { + "stringValue": "grant-ben-sentry-root" + } + }, + { + "key": "seats.scope.audience", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "u-ana" + }, + { + "stringValue": "u-ben" + } + ] + } + } + }, + { + "key": "seats.scope.budget.calls", + "value": { + "intValue": "20" + } + }, + { + "key": "seats.scope.budget.window_ms", + "value": { + "intValue": "3600000" + } + }, + { + "key": "seats.scope.connection_id", + "value": { + "stringValue": "conn-ben-sentry" + } + }, + { + "key": "seats.scope.tools", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "sentry.get_issue" + }, + { + "stringValue": "sentry.resolve_issue" + } + ] + } + } + }, + { + "key": "seats.scope.write_mode", + "value": { + "stringValue": "write-with-confirm" + } + }, + { + "key": "seats.target.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.target.kind", + "value": { + "stringValue": "seat" + } + } + ] + }, + { + "timeUnixNano": "1792944004000000000", + "name": "credential.granted", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "3" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.names", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "OPENROUTER_API_KEY" + }, + { + "stringValue": "SENTRY_PROXY_TOKEN" + } + ] + } + } + } + ] + }, + { + "timeUnixNano": "1792944012000000000", + "name": "liveness.recovery", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "11" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.action", + "value": { + "stringValue": "retry" + } + }, + { + "key": "seats.attempt", + "value": { + "intValue": "2" + } + } + ] + }, + { + "timeUnixNano": "1792944029000000000", + "name": "result.reported", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "28" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.refs", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "id", + "value": { + "stringValue": "#212" + } + }, + { + "key": "type", + "value": { + "stringValue": "pr" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "done" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "f213e9d395dc7277", + "parentSpanId": "4f7ea2299f50ef90", + "name": "invoke_agent Ana's opencode", + "kind": 1, + "startTimeUnixNano": "1792944005000000000", + "endTimeUnixNano": "1792944011000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.19.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "stuck" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "stuck" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944006000000000", + "name": "turn.prompt_sent", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "5" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.chars", + "value": { + "intValue": "412" + } + } + ] + }, + { + "timeUnixNano": "1792944009000000000", + "name": "liveness.stalled", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "8" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.idle_ms", + "value": { + "intValue": "90000" + } + }, + { + "key": "seats.window_ms", + "value": { + "intValue": "90000" + } + } + ] + }, + { + "timeUnixNano": "1792944010000000000", + "name": "liveness.recovery", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "9" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.action", + "value": { + "stringValue": "cancel" + } + }, + { + "key": "seats.attempt", + "value": { + "intValue": "1" + } + } + ] + } + ], + "status": { + "code": 2, + "message": "stuck" + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "a95791040c71e9f0", + "parentSpanId": "f213e9d395dc7277", + "name": "execute_tool Read src/checkout/total.js", + "kind": 1, + "startTimeUnixNano": "1792944007000000000", + "endTimeUnixNano": "1792944008000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "Read src/checkout/total.js" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-1" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "native" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944008000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "7" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "54966552492d24ee", + "parentSpanId": "4f7ea2299f50ef90", + "name": "invoke_agent Ana's opencode", + "kind": 1, + "startTimeUnixNano": "1792944013000000000", + "endTimeUnixNano": "1792944028000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "ses-ana-2" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "opencode" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.19.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "18234" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "2210" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944027000000000", + "name": "usage", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "26" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.input_tokens", + "value": { + "intValue": "18234" + } + }, + { + "key": "seats.output_tokens", + "value": { + "intValue": "2210" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "109cfe1b8a44b804", + "parentSpanId": "54966552492d24ee", + "name": "execute_tool git push origin fix/total-rounding", + "kind": 1, + "startTimeUnixNano": "1792944014000000000", + "endTimeUnixNano": "1792944017000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "git push origin fix/total-rounding" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-2" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "execute" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "native" + } + }, + { + "key": "error.type", + "value": { + "stringValue": "tool_failed" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944015000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "14" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + }, + { + "timeUnixNano": "1792944016000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "15" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "reject" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "deny" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "execute:git push*" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + }, + { + "timeUnixNano": "1792944017000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "16" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "failed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-2" + } + } + ] + } + ], + "status": { + "code": 2, + "message": "tool_failed" + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "b3182723ae254b59", + "parentSpanId": "54966552492d24ee", + "name": "execute_tool sentry.resolve_issue", + "kind": 1, + "startTimeUnixNano": "1792944018000000000", + "endTimeUnixNano": "1792944021000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "sentry.resolve_issue" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-3" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "fetch" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "connector" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944019000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "18" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + }, + { + "timeUnixNano": "1792944020000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "19" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.approval", + "value": { + "stringValue": "approved" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "credential-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-7f3" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "ask" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "fetch" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + }, + { + "timeUnixNano": "1792944021000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "20" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-3" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "400ad84d51bc8edf", + "parentSpanId": "54966552492d24ee", + "name": "execute_tool commonly_post_message", + "kind": 1, + "startTimeUnixNano": "1792944022000000000", + "endTimeUnixNano": "1792944025000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "commonly_post_message" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-4" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "other" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "commonly" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944023000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "22" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + }, + { + "timeUnixNano": "1792944024000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "23" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "other:commonly_*" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + }, + { + "timeUnixNano": "1792944025000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "24" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ana-opencode" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ana's opencode" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-4" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "a4aa63e7490207ea", + "name": "seats.run", + "kind": 1, + "startTimeUnixNano": "1792944032000000000", + "endTimeUnixNano": "1792944043000000000", + "attributes": [ + { + "key": "seats.run.id", + "value": { + "stringValue": "run-ben-0001" + } + }, + { + "key": "seats.task.id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.pod.id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.run.outcome", + "value": { + "stringValue": "ok" + } + }, + { + "key": "seats.run.attempts", + "value": { + "intValue": "1" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944032000000000", + "name": "grant.issued", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.binding.ends_on", + "value": { + "stringValue": "task.done" + } + }, + { + "key": "seats.binding.pod_id", + "value": { + "stringValue": "pod-checkout" + } + }, + { + "key": "seats.binding.task_id", + "value": { + "stringValue": "TASK-101" + } + }, + { + "key": "seats.expires_at", + "value": { + "stringValue": "2026-10-25T20:00:00.000Z" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, + { + "key": "seats.issued_by", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.scope.audience", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "u-ben" + } + ] + } + } + }, + { + "key": "seats.scope.connection_id", + "value": { + "stringValue": "conn-ben-github" + } + }, + { + "key": "seats.scope.tools", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "github.get_pull_request" + } + ] + } + } + }, + { + "key": "seats.scope.write_mode", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.target.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.target.kind", + "value": { + "stringValue": "seat" + } + } + ] + }, + { + "timeUnixNano": "1792944033000000000", + "name": "credential.granted", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "2" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "seats" + } + }, + { + "key": "seats.names", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "OPENAI_API_KEY" + } + ] + } + } + } + ] + }, + { + "timeUnixNano": "1792944041000000000", + "name": "task.done", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "10" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "board" + } + }, + { + "key": "seats.done_by", + "value": { + "stringValue": "u-ben" + } + } + ] + }, + { + "timeUnixNano": "1792944042000000000", + "name": "grant.revoked", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "11" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "system" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "board" + } + }, + { + "key": "seats.grant_ids", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "grant-ben-review-1" + } + ] + } + } + }, + { + "key": "seats.reason", + "value": { + "stringValue": "task.done" + } + } + ] + } + ], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "5fd2950fcbddd334", + "parentSpanId": "a4aa63e7490207ea", + "name": "invoke_agent Ben's goose", + "kind": 1, + "startTimeUnixNano": "1792944034000000000", + "endTimeUnixNano": "1792944040000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "invoke_agent" + } + }, + { + "key": "gen_ai.agent.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.agent.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.attempt.number", + "value": { + "intValue": "1" + } + }, + { + "key": "seats.adapter", + "value": { + "stringValue": "goose" + } + }, + { + "key": "seats.cli.name", + "value": { + "stringValue": "goose" + } + }, + { + "key": "seats.cli.version", + "value": { + "stringValue": "1.54.0" + } + }, + { + "key": "seats.attempt.outcome", + "value": { + "stringValue": "ok" + } + } + ], + "events": [], + "status": { + "code": 0 + } + }, + { + "traceId": "d043bdec4346bb1253c06463a2c2cc13", + "spanId": "d45b1520df0f2e72", + "parentSpanId": "5fd2950fcbddd334", + "name": "execute_tool github.get_pull_request", + "kind": 1, + "startTimeUnixNano": "1792944035000000000", + "endTimeUnixNano": "1792944038000000000", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "github.get_pull_request" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "tc-1" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.tool.kind", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool.surface", + "value": { + "stringValue": "connector" + } + } + ], + "events": [ + { + "timeUnixNano": "1792944036000000000", + "name": "permission.requested", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "5" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.options", + "value": { + "arrayValue": { + "values": [ + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "allow" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "allow_always" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "always" + } + } + ] + } + }, + { + "kvlistValue": { + "values": [ + { + "key": "kind", + "value": { + "stringValue": "reject_once" + } + }, + { + "key": "option_id", + "value": { + "stringValue": "reject" + } + } + ] + } + } + ] + } + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + }, + { + "timeUnixNano": "1792944037000000000", + "name": "permission.decided", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "6" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "grant:grant-ben-review-1" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, + { + "key": "seats.kind", + "value": { + "stringValue": "allow_once" + } + }, + { + "key": "seats.option_id", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.outcome", + "value": { + "stringValue": "selected" + } + }, + { + "key": "seats.rule.list", + "value": { + "stringValue": "allow" + } + }, + { + "key": "seats.rule.pattern", + "value": { + "stringValue": "read" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + }, + { + "timeUnixNano": "1792944038000000000", + "name": "tool.update", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "7" + } + }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "agent" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-agent-ben-goose" + } + }, + { + "key": "seats.actor.owner", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.actor.name", + "value": { + "stringValue": "Ben's goose" + } + }, + { + "key": "seats.status", + "value": { + "stringValue": "completed" + } + }, + { + "key": "seats.tool_call_id", + "value": { + "stringValue": "tc-1" + } + } + ] + } + ], + "status": { + "code": 0 + } + } + ] + } + ] + } + ] +} diff --git a/packages/seats/docs/examples/run-ana-0001.jsonl b/packages/seats/docs/examples/run-ana-0001.jsonl new file mode 100644 index 0000000000..902db3a75a --- /dev/null +++ b/packages/seats/docs/examples/run-ana-0001.jsonl @@ -0,0 +1,29 @@ +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"leaseUntil":"2026-10-25T18:00:00Z"},"kind":"task.claimed","podId":"pod-checkout","prev":null,"runId":"run-ana-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:02.000Z","v":1} +{"actor":{"id":"u-ana","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ana-sentry-1","issuedBy":"u-ana","parentGrantId":"grant-ben-sentry-root","scope":{"audience":["u-ana","u-ben"],"budget":{"calls":20,"windowMs":3600000},"connectionId":"conn-ben-sentry","tools":["sentry.get_issue","sentry.resolve_issue"],"writeMode":"write-with-confirm"},"target":{"id":"u-agent-ana-opencode","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":"112fd59d4b22a34212a1051217ea50cbee7fc5756d1db32c0487760e84e0b2c2","runId":"run-ana-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:03.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENROUTER_API_KEY","SENTRY_PROXY_TOKEN"]},"kind":"credential.granted","podId":"pod-checkout","prev":"02b60a920d4e46710f1da5d478472e32054abef954f1921f836cfb7b8f2466b2","runId":"run-ana-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:04.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":1,"cli":{"name":"opencode","version":"1.19.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"8de2801011b74ef1386e35eaa53bf6e0d5fce3692309ba832bc44a9005c2b97c","runId":"run-ana-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:05.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"chars":412},"kind":"turn.prompt_sent","podId":"pod-checkout","prev":"2191f906c453d8adbf118b25a096134313c4db9f2776c9d81462eef066c80a88","runId":"run-ana-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:06.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"read","status":"pending","surface":"native","title":"Read src/checkout/total.js","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"53d2f32e2e78ce199c9b84eba5140d79b62b01f2ef8de43acd527e98739d43a3","runId":"run-ana-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:07.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"40c3dfe7e66cec5306e69ed96798950b8d1a1aa8e1dd6ad8c8384439b99f7ce3","runId":"run-ana-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:08.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"idleMs":90000,"windowMs":90000},"kind":"liveness.stalled","podId":"pod-checkout","prev":"212b5d465b8936032324127c8b834523df956bc6bc5fde15004d0b4932009dfb","runId":"run-ana-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:09.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"action":"cancel","attempt":1},"kind":"liveness.recovery","podId":"pod-checkout","prev":"260faaf14f1c7bf121338f251a1c88e41c78689ccdbfa0fe3f36831bda5c3848","runId":"run-ana-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:10.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"failure":{"detail":"no progress signal for 90000ms (window 90000ms)","kind":"stuck"},"outcome":"stuck"},"kind":"attempt.settled","podId":"pod-checkout","prev":"c7637334d2a2bdeea8d5d6ae3b402d6d348bcd18ee563cbe2ef00576cb7c44ad","runId":"run-ana-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:11.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"action":"retry","attempt":2},"kind":"liveness.recovery","podId":"pod-checkout","prev":"0ee1be8a38ec355ebd500c99c09517e2607ab01a61ef6989770bb09a1b1f3adf","runId":"run-ana-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:12.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":2,"cli":{"name":"opencode","version":"1.19.0"},"sessionId":"ses-ana-2"},"kind":"attempt.started","podId":"pod-checkout","prev":"222b2d342255178de812757d277bf1913c14ee21178b7f67da8c2ddc1342ed42","runId":"run-ana-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:13.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"execute","status":"pending","surface":"native","title":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"tool.call","podId":"pod-checkout","prev":"ad5d76d6df5f3aa0a6c914602248135ff230fb75f3abc79cace9ff5094b257d0","runId":"run-ana-0001","schema":"seats.record","seq":13,"taskId":"TASK-101","ts":"2026-10-25T16:00:14.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-2"},"kind":"permission.requested","podId":"pod-checkout","prev":"34c7eb2c6facb2bb29bef8c63443eac1179a1b3e80bf26d4425df090b686303a","runId":"run-ana-0001","schema":"seats.record","seq":14,"taskId":"TASK-101","ts":"2026-10-25T16:00:15.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"reject_once","optionId":"reject","outcome":"selected","rule":{"list":"deny","pattern":"execute:git push*"},"toolCallId":"tc-2"},"kind":"permission.decided","podId":"pod-checkout","prev":"c51c441f24bc1126084d5918d180bb0735c9aab07b3756426cdf274333c38193","runId":"run-ana-0001","schema":"seats.record","seq":15,"taskId":"TASK-101","ts":"2026-10-25T16:00:16.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"failed","toolCallId":"tc-2"},"kind":"tool.update","podId":"pod-checkout","prev":"46f1eeb72be62623c2b8605a6115370207f11ccf1edccfc429cfe641edd37e50","runId":"run-ana-0001","schema":"seats.record","seq":16,"taskId":"TASK-101","ts":"2026-10-25T16:00:17.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"fetch","name":"sentry.resolve_issue","status":"pending","surface":"connector","title":"sentry.resolve_issue CHECKOUT-88","toolCallId":"tc-3"},"kind":"tool.call","podId":"pod-checkout","prev":"6f59e269f201abfa36dc338fe094a287fcee110cf734df26b8072db46a7476fe","runId":"run-ana-0001","schema":"seats.record","seq":17,"taskId":"TASK-101","ts":"2026-10-25T16:00:18.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-3"},"kind":"permission.requested","podId":"pod-checkout","prev":"259adaed1cdf12c494fd150cb604a54f51b7780154f87eba8ad40d2796d1a1d0","runId":"run-ana-0001","schema":"seats.record","seq":18,"taskId":"TASK-101","ts":"2026-10-25T16:00:19.000Z","v":1} +{"actor":{"id":"u-ben","type":"human"},"data":{"approval":"approved","decidedBy":"u-ben","decidedByRole":"credential-owner","decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"ask","pattern":"fetch"},"toolCallId":"tc-3"},"kind":"permission.decided","podId":"pod-checkout","prev":"bbf582d7342a8966424e1ff04805088c8468f2ac5d55dfc71c5f714bd6d183cc","runId":"run-ana-0001","schema":"seats.record","seq":19,"taskId":"TASK-101","ts":"2026-10-25T16:00:20.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-3"},"kind":"tool.update","podId":"pod-checkout","prev":"9e2395f74135669ef532fcb750602ca8617803ac8d061a1b5680a430be1ba687","runId":"run-ana-0001","schema":"seats.record","seq":20,"taskId":"TASK-101","ts":"2026-10-25T16:00:21.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"other","name":"commonly_post_message","status":"pending","surface":"commonly","title":"commonly_post_message","toolCallId":"tc-4"},"kind":"tool.call","podId":"pod-checkout","prev":"3257330abfb7e0e9e9be4bc549df79e7da6c3d154e6b4300bbba9abb965fd976","runId":"run-ana-0001","schema":"seats.record","seq":21,"taskId":"TASK-101","ts":"2026-10-25T16:00:22.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-4"},"kind":"permission.requested","podId":"pod-checkout","prev":"e3d66ce0c7ab8dd4bf87583f2bb15d1d8a3ff1b84a12820f975041f65114aec7","runId":"run-ana-0001","schema":"seats.record","seq":22,"taskId":"TASK-101","ts":"2026-10-25T16:00:23.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"other:commonly_*"},"toolCallId":"tc-4"},"kind":"permission.decided","podId":"pod-checkout","prev":"973c505153298862657039b572aeb0cb27bc86030d3dd220ad146437f7689da1","runId":"run-ana-0001","schema":"seats.record","seq":23,"taskId":"TASK-101","ts":"2026-10-25T16:00:24.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-4"},"kind":"tool.update","podId":"pod-checkout","prev":"d9a5a1169bd53ad6c60bf10e3cb7ef1a7a36a93b4a5b2c9ef442df999d52825d","runId":"run-ana-0001","schema":"seats.record","seq":24,"taskId":"TASK-101","ts":"2026-10-25T16:00:25.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"text":"Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"f4fa58c0c29f4c988950042cc5c42b25355ec0cb00b9f6bf3abc5011cd63e88f","runId":"run-ana-0001","schema":"seats.record","seq":25,"taskId":"TASK-101","ts":"2026-10-25T16:00:26.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"inputTokens":18234,"outputTokens":2210},"kind":"usage","podId":"pod-checkout","prev":"12364a5e040c116f6acbceb87cea8c712853c5b4bada8680fffab671eeaf223b","runId":"run-ana-0001","schema":"seats.record","seq":26,"taskId":"TASK-101","ts":"2026-10-25T16:00:27.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":2,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"24b24a0f64242aacf7b78097b131eba62c469d35c2125453136df6c5e87853eb","runId":"run-ana-0001","schema":"seats.record","seq":27,"taskId":"TASK-101","ts":"2026-10-25T16:00:28.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"refs":[{"id":"#212","type":"pr"}],"status":"done"},"kind":"result.reported","podId":"pod-checkout","prev":"caad0b53230898f529c5f0c332a12763193c7786c190603cd7774fa5db59ed1e","runId":"run-ana-0001","schema":"seats.record","seq":28,"taskId":"TASK-101","ts":"2026-10-25T16:00:29.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":2,"durationMs":412000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"c3c79cddb97be9fcbd02bcaf8081b23026c78caa9d3810c73f79d1c9b79b52fb","runId":"run-ana-0001","schema":"seats.record","seq":29,"taskId":"TASK-101","ts":"2026-10-25T16:00:30.000Z","v":1} diff --git a/packages/seats/docs/examples/run-ben-0001.jsonl b/packages/seats/docs/examples/run-ben-0001.jsonl new file mode 100644 index 0000000000..60ad22d6c3 --- /dev/null +++ b/packages/seats/docs/examples/run-ben-0001.jsonl @@ -0,0 +1,12 @@ +{"actor":{"id":"u-ben","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ben-review-1","issuedBy":"u-ben","scope":{"audience":["u-ben"],"connectionId":"conn-ben-github","tools":["github.get_pull_request"],"writeMode":"read"},"target":{"id":"u-agent-ben-goose","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":null,"runId":"run-ben-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:32.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENAI_API_KEY"]},"kind":"credential.granted","podId":"pod-checkout","prev":"f1ad10d77360347b5a51560adb0aad2d3b2dd8dcb872fb962b83e06f1735e2d4","runId":"run-ben-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:33.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"adapter":"goose","attempt":1,"cli":{"name":"goose","version":"1.54.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"54299a711ac2afca451e8be0574218f91f02ce64041b1097f8cd29f3e169ce5f","runId":"run-ben-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:34.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"kind":"read","name":"github.get_pull_request","status":"pending","surface":"connector","title":"github.get_pull_request #212","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"16c619cb9a9fb8f760186168a22685f2ec6708948a9ff70e35294140921dcccf","runId":"run-ben-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:35.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-1"},"kind":"permission.requested","podId":"pod-checkout","prev":"77ee7b8804e3d60b7d34050b10df34bd8b0d8ce85a4cbdc48bc57d6c099edf40","runId":"run-ben-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:36.000Z","v":1} +{"actor":{"id":"grant:grant-ben-review-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ben-review-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"read"},"toolCallId":"tc-1"},"kind":"permission.decided","podId":"pod-checkout","prev":"dec1718f26e8617628e7f3c522ab23a6a4601802920b2723c2cae700be3de62a","runId":"run-ben-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:37.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"a00470b2fb1d368d5210df55e829815c8692a546347731e033687aca5396826d","runId":"run-ben-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:38.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"text":"Reviewed #212: the rounding fix is right and the half-cent case is tested."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"8ab27c571e0f78429a09a77d01d2ff63ec2052503b0d0a49207056719f416949","runId":"run-ben-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:39.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"2df713d5685675553f11676224b674459da8cf14b389879c9e4ec35a203768b5","runId":"run-ben-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:40.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"doneBy":"u-ben"},"kind":"task.done","podId":"pod-checkout","prev":"029be6e1a3f91e8309ef92e4c5faf6c89fb22eb8fb087f8ef4b11f6a45167803","runId":"run-ben-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:41.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"grantIds":["grant-ben-review-1"],"reason":"task.done"},"kind":"grant.revoked","podId":"pod-checkout","prev":"647595a7d6a12380b8b004caefb6ac2f48c7f287fe70f8a0e6862f85dffa14f1","runId":"run-ben-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:42.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":1,"durationMs":95000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"c631427d4c505347aeacc1135eeab6519eaf8fdec9d9302c46c79aa78c6795de","runId":"run-ben-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:43.000Z","v":1} diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md new file mode 100644 index 0000000000..879fde3db8 --- /dev/null +++ b/packages/seats/docs/record-v1.md @@ -0,0 +1,431 @@ +# Run record, format v1 + +A run record is an append-only log of one agent run: +- who claimed which task; +- which access the run was granted, and by whom; +- which tool calls were asked about, and who decided them; +- what the agent attempted, and what came of it. + +The format is plain JSON lines, so a tool that has never seen this code can +read it, verify it and export it. + +One file holds one run. Several runs of the same task (a retry under a new +run, another person's agent reviewing the work) are several files that share +a `podId` and a `taskId`. + +The reference implementation: +- `src/record/format.js` writes and verifies records; +- `src/record/otel.js` exports them as OTLP; +- `src/trace/store.js` is the store that writes them. + +A generated sample is in [`examples/`](examples/) (section 7). + +## 1. Envelope + +Each line is one JSON object, followed by `\n` (UTF-8): + +```json +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":1,"cli":{"name":"opencode","version":"1.19.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"8de2801011b74ef1386e35eaa53bf6e0d5fce3692309ba832bc44a9005c2b97c","runId":"run-ana-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:05.000Z","v":1} +``` + +| field | type | meaning | +|---|---|---| +| `v` | integer | Format version. This document is `1`. | +| `schema` | string | Always `"seats.record"`. | +| `runId` | string | The run, `[A-Za-z0-9_-]{6,64}`. The same on every line of a file. | +| `taskId` | string, optional | The board's task id, verbatim (e.g. `"TASK-490"`). The same on every line when present. | +| `podId` | string; **required when `taskId` is set** | The pod the run belongs to. Task ids restart in each pod, so `taskId` is unique only together with `podId`. The same on every line when present. | +| `seq` | integer | 1 on the first line, then +1 per line, with no gaps. | +| `ts` | string | RFC 3339 timestamp in UTC: `YYYY-MM-DDTHH:MM:SS(.fraction)Z`. | +| `kind` | string | What happened (section 2). | +| `actor` | object or `null` | Who caused the entry (below). | +| `data` | object | Kind-specific fields (section 2). | +| `prev` | string or `null` | Lowercase hex SHA-256 of the previous line's bytes. `null` on `seq` 1. | + +**`actor`** is `{type, id, owner?, name?}`: +- `type` is `agent`, `human`, `policy` or `system`. +- `id` is an identity, never a display name. + - For `agent` and `human` it is the instance's user id. Agents are users too. + - For `system` it names the writing component: `seats` (the run writer) + or `board`. + - For `policy` it names the grant whose rules decided (`grant:<grantId>`), + or `policy`. + - It is a token (`[A-Za-z0-9][A-Za-z0-9._:-]{0,127}`), so a name with spaces + can't pass for one. +- `owner` is set on `agent` actors only. It is the user id of the person the + agent belongs to. +- `name` is an optional display name. It is never part of the identity. + +The owner of an agent and the person who approves what it does are separate +fields on purpose: +- `actor.owner` says whose agent this is; +- the approver is `decidedBy` on `permission.decided`. + +In the case this format exists for, they differ: Ana's agent asks, and Ben +approves. + +`actor: null` means the writer did not know who caused the entry. A reader +must not fill it in. Each kind has a default actor (section 2), which a +writer records when the emitter names none. + +**Canonical serialization.** A line is the JSON Canonicalization Scheme +(RFC 8785) form of its entry: +- object keys are sorted by UTF-16 code units; +- there is no insignificant whitespace; +- strings and numbers are serialized as ECMAScript `JSON.stringify` does; +- an absent field is omitted, never written as `null`. `actor` and `prev` are + the two fields where `null` is a value. + +A writer in any language that follows these rules produces byte-identical +lines. A verifier rejects a line that is not in canonical form. + +**What `prev` covers.** `prev` is SHA-256 over the exact bytes of the previous +line, without its trailing `\n`. It is not taken over a re-serialization, so +a verifier needs no JSON library to check the chain. + +## 2. Kinds + +Every kind below is part of v1. "Default actor" is the actor a writer records +when the emitter names none. "run" means the run's own actor, usually the +agent. + +### Claim + +The board is the source of truth for tasks. Its states are `pending`, +`claimed`, `done` and `blocked`. The record only observes the board: these +entries mirror what the board already holds, and they never change it. + +| kind | data | default actor | +|---|---|---| +| `task.claimed` | `{leaseUntil?}` | run | +| `task.released` | `{reason?}`, such as `blocked`, `lease_expired`, `reassigned` or `abandoned` | run | +| `task.done` | `{doneBy?}` (a user id) | `{type: system, id: board}`. The board is its writer. | + +`task.done` needs a run with a `taskId`. Writing it ends every open grant +bound to that task (below). + +### Grant + +A grant has the shape of a room grant: a connection, exact tool names, a +write mode, a budget and an audience, plus its parent for attenuation. + +| kind | data | default actor | +|---|---|---| +| `grant.issued` | `{grantId, parentGrantId?, scope, target?, issuedBy?, expiresAt?, binding?}` | derived from `issuedBy`: `policy` gives a policy actor, a user id gives a human actor | +| `grant.revoked` | `{grantIds: [grantId, …], reason}` | `system` | + +**`scope`** is `{connectionId?, tools: [name], writeMode?, budget?, audience: [userId]}`: +- `connectionId` is the connection whose credential the grant uses. The + credential itself is never named or recorded. A broker resolves it. +- `tools` are exact tool names, such as `sentry.resolve_issue`, never patterns. +- `writeMode` is `read`, `write` or `write-with-confirm`. +- `budget` is `{calls?, windowMs?}`. +- `audience` lists the user ids the grant was minted for. + +**The other fields:** +- `parentGrantId` is the grant this one attenuates, if any. Following it gives + the lineage. +- `target` is `{kind: pod|seat, id}`: whom the grant is to. +- `issuedBy` is a user id, or `policy`. +- `binding` is `{taskId, podId, endsOn}`, and it must name the run's own + `taskId` and `podId`. `endsOn` is one of: + - `task.done`: the grant ends when the board marks its task done; + - `expiresAt`: it ends at `expiresAt`, which is then required. + +**`grant.revoked`** ends every id in `grantIds` in a single entry. `reason` is +one of: +- `task.done`: the bound task is done. The writer records this right after `task.done`; +- `expired`: `expiresAt` passed; +- `cascade`: an ancestor was revoked. `grantIds[0]` is that ancestor, and the + rest are its descendants, all ended by the same write; +- `manual`: a person or policy revoked exactly these grants. The actor says who. + +**What revocation means in v1.** A revoked grant is no longer valid. A +credential that a run placed in its agent process's environment stays there +until that process exits. A revocation records that the grant's term ended; +it does not prove the process lost access. That happens at `run.settled`. + +### Approval + +| kind | data | default actor | +|---|---|---| +| `permission.requested` | `{toolCallId?, options: [{optionId, kind}]}` | run | +| `permission.decided` | `{outcome, optionId?, kind?, toolCallId, rule, approval?, decidedBy, decidedByRole, decisionCardId?, grantId?}` | the decider: a human actor for a user id, a policy actor for `policy` | + +**The decision:** +- `outcome` is `selected` or `cancelled`. +- `kind` is the ACP option kind chosen: `allow_once` approves, and + `reject_once` refuses. +- `rule` is `{list, pattern}`: + - `list` is `allow`, `ask`, `deny` or `default` (no rule matched); + - `pattern` is the run policy's pattern that matched, or `null`. +- `approval` is present only when someone was asked. It is `approved`, + `rejected`, `timed_out`, `failed` or `aborted`. + +**Who decided:** +- `decidedBy` is a user id, `"policy"`, or `null`. + - `null` means the approver answered without saying who it was. + - A rule-decided call, or the default given when no answer came in time, is + `"policy"`. +- `decidedByRole` is one of: + - `credential-owner`: the owner of the connection the call would use. This + is who decides approvals today; + - `decision-owner`: a person assigned to the decision. This is reserved, + and writers may not emit it yet; + - `agent-owner`: the owner of the asking agent; + - `policy`; + - `null`, when unknown. +- `decisionCardId` is set when the decision came through a decision card. +- `grantId` ties the decision to its `grant.issued`. + +### Attempt + +| kind | data | default actor | +|---|---|---| +| `attempt.started` | `{attempt, adapter, cli?: {name, version}, sessionId?}` | run | +| `attempt.settled` | `{attempt, outcome, failure?: {kind, detail}}` | `system` | +| `run.settled` | `{outcome, failure?, durationMs, attempts}` | `system` | + +- `outcome` is `ok`, `failed`, `stuck` or `cancelled`. +- `failure.kind` is a key of `FAILURE_KINDS` in `src/failures.js`, or `null` + when the cause had no name. +- The agent's id and owner are the actor of `attempt.started`. +- `run.settled` is the last line of a complete record. Nothing is written + after it. +- If an attempt has no `attempt.settled`, the record says only that the next + `attempt.started`, or `run.settled`, came after it. A reader must not infer + the outcome. + +### Tool activity + +| kind | data | default actor | +|---|---|---| +| `tool.call` | `{toolCallId, title, kind?, status?, name?, surface?}` | run | +| `tool.update` | `{toolCallId, status?, title?}` | run | + +- `kind` is the ACP tool kind (`read`, `edit`, `execute`, `fetch`, …). +- `name` is the tool's own name, when known. +- `surface` says where the call went: + - `connector`: a third-party service, through a connector; + - `commonly`: the room's own tools (`commonly_*`); + - `native`: the CLI's built-in tools. + + All three land in the same record. + +### Credentials + +| kind | data | default actor | +|---|---|---| +| `credential.granted` | `{names: [NAME]}` | `system` | +| `credential.denied` | `{name, reason}`, where `reason` is `not_granted` or `unresolved` | `system` | + +These cover the environment credentials a run is started with. Values never +appear here: `NAME` is an environment variable name matching +`[A-Z_][A-Z0-9_]*`. + +### Result + +| kind | data | default actor | +|---|---|---| +| `result.reported` | `{status, refs?: [{type, id}]}` | run | + +- `status` is `done`, `partial`, `blocked` or `failed`. +- A ref points at what the run produced, such as `{type: "pr", id: "#212"}`. + +### Run activity + +These kinds carry the trace vocabulary through unchanged. Their data is +documented in `src/trace/events.js`. + +| kind | default actor | +|---|---| +| `turn.prompt_sent` | `system` | +| `agent.message_chunk`, `agent.thought_chunk`, `plan`, `usage` | run | +| `liveness.stalled`, `liveness.recovery`, `process.exited` | `system` | + +### Mapping from the trace vocabulary + +Emitters keep calling `emit(type, data)` with `EVENT_TYPES` names, and the +store writes the record kind: + +| `EVENT_TYPES` entry | record kind | +|---|---| +| `run.started` | `attempt.started` (renamed: it is emitted once per attempt) | +| every other entry | the same name | + +What's new or changed in `EVENT_TYPES`: +- `task.claimed`, `task.released`, `task.done`, `grant.issued`, + `grant.revoked`, `attempt.settled` and `result.reported` are new entries. +- `emit` also accepts the record kind `attempt.started`. +- `readRun` returns each entry with its trace name as `type` (`run.started` + for an `attempt.started` line), and puts the envelope fields beside it. + +## 3. Rules + +1. **No secret values, by construction.** + - No field of any kind holds a credential value. A grant names a + connection id. A run's environment credentials appear only as names, + and a writer refuses one that is not name-shaped. + - Agent-authored text, such as message chunks and tool titles, belongs to + the agent and could repeat a value the agent holds. A writer that knows + the run's credential values replaces every occurrence of 8 characters or + more with `[redacted]` before the line is hashed. + - That replacement covers known values only. A value the writer was not + given, or one the agent transformed (base64, split across chunks), passes + through. +2. **Readers ignore unknown fields and skip unknown kinds.** A reader that + meets a kind it doesn't know still checks the chain over that line. +3. **`v` bumps only on a breaking change:** + - a field removed or renamed; + - a meaning changed; + - a change to the hash or serialization rules. + + Adding a kind, an optional field or an enum value doesn't break anything, + and keeps `v: 1`. A reader refuses a `v` higher than it knows. +4. **One writer per file.** A single process appends lines in `seq` order. + +## 4. What the hash chain proves, and what it doesn't + +**It detects:** +- an edited byte in any line except the last; +- a deleted line, which leaves a `seq` gap and a `prev` mismatch; +- lines reordered within the file; +- a torn final line, where a crash stopped a write partway. + +**The last line** is covered only against an outside record of the head: +- When a run settles, the store writes `head` (the SHA-256 of the last line) + and `entries` to its index, and `verifyRun` checks both. +- The index sits in the same directory and is just as editable, so this + catches accidents, not an adversary. +- A record cut short before `run.settled` is a valid prefix. It is reported + as unsettled, not as tampered. + +**It does not prove:** +- that any event happened. The writer could have recorded anything; +- who wrote the file. There is no signature in v1; +- that the whole file wasn't replaced by a different chain that is consistent + with itself; +- anything across files. Two runs of one task are not chained together. + +So the chain shows that the file is the one its writer wrote, in the order +it was written. Whether the writer told the truth is a question for whoever +operates it. + +## 5. OpenTelemetry mapping + +`toOtlpJson(entries, {serviceName})` returns an OTLP/JSON +`ExportTraceServiceRequest`, nested as `resourceSpans` → `scopeSpans` → +`spans`. The resource carries `service.name`, and the scope is +`seats.record`, version `1`. + +### Shape + +- **A task is a trace.** `traceId` is the first 16 bytes of + SHA-256(`"seats.record/trace/task/" + podId + "/" + taskId`). A run with no + task uses SHA-256(`"seats.record/trace/run/" + runId`). Every run of one + task lands in one trace. +- **A run is a span:** `seats.run`, kind INTERNAL, the root of its run. +- **An attempt is a child span of its run:** `invoke_agent {agent name}`. +- **A tool call is a child span of its attempt:** `execute_tool {tool name}`. +- **Decisions are span events.** A `permission.*` entry goes on the tool span + with the same `toolCallId`. Without one, it goes on the current attempt, + then the run. +- **Every other kind is an event on the span current at the time.** That + covers grants, credentials, claims, `task.done`, results, liveness and + `tool.update`. +- **Span ids** are the first 8 bytes of + SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to + the entry that opens the span. The run span uses `0`. Exporting the same + entries always produces the same bytes. +- **Times** come from the entries' `ts`: + - A span ends at its closing entry: `attempt.settled`, the next + `attempt.started`, `run.settled`, or a tool's `completed` or `failed` + update. + - A span with no closing entry ends when its parent does. +- **Status:** + - The outcomes `failed`, `stuck` and `cancelled`, and a tool update with + status `failed`, set `STATUS_CODE_ERROR`. + - The error class goes in `error.type`: the failure kind, or `tool_failed`. + - Success leaves the status unset, as OpenTelemetry recommends for + instrumentation. +- **Not exported:** message and thought chunks are left out, following the + upstream conventions, which make content capture opt-in. Unknown kinds are + skipped. + +### OpenTelemetry GenAI conventions used + +The source is github.com/open-telemetry/semantic-conventions-genai, read on +2026-10-25 at commit `9ffc7e00`. The GenAI pages on opentelemetry.io now say +they have moved there. + +**Upstream status:** both documents used here (`gen-ai-agent-spans.md` and +`gen-ai-spans.md`), and every `gen_ai.*` attribute below, are **Development** +status. They can change without a major version, so this mapping is pinned +to that commit. `error.type` is Stable. + +| attribute | on | from | +|---|---|---| +| `gen_ai.operation.name` | attempt, tool | `invoke_agent`, `execute_tool` | +| `gen_ai.agent.id` | attempt | `actor.id` of `attempt.started` | +| `gen_ai.agent.name` | attempt, tool | `actor.name`, else `actor.id` | +| `gen_ai.conversation.id` | attempt | `sessionId` | +| `gen_ai.tool.name` | tool | `name`, else `title` | +| `gen_ai.tool.call.id` | tool | `toolCallId` | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | attempt | the last `usage` entry in the attempt | +| `error.type` | any span with error status | failure kind, or `tool_failed` | + +### Choices and gaps + +- **The attempt span is INTERNAL, not CLIENT.** The agent runs in a child + process, so neither upstream variant fits exactly. The CLIENT variant of + `invoke_agent` makes `gen_ai.provider.name` Required, and a supervisor + cannot know which provider a CLI chose. +- **`gen_ai.tool.type` (Recommended) is not set.** Its upstream values + (`function`, `extension`, `datastore`) don't map cleanly onto `surface`. +- **Everything else uses the neutral `seats.*` namespace:** + - these named attributes: `seats.run.id`, `seats.task.id`, `seats.pod.id`, + `seats.run.outcome`, `seats.run.attempts`, `seats.attempt.number`, + `seats.attempt.outcome`, `seats.adapter`, `seats.cli.name`, + `seats.cli.version`, `seats.agent.owner`, `seats.tool.kind` and + `seats.tool.surface`; + - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner` and + `.name`; + - the event's data, flattened as `seats.<path>` with camelCase segments in + snake_case. For example, `decidedBy` becomes `seats.decided_by`, and + `scope.connectionId` becomes `seats.scope.connection_id`. + +## 6. Not in v1 (open for v2) + +- **Signing.** The writer signs each line, or each head, so that "who wrote + this" can be proved and a replaced file can be detected. +- **Multi-writer chains.** A pod-level record that several hosts append to + needs ordering across writers: one chain per writer plus a merge, or a + sequencer. +- **Cross-run linking.** A run's first line could be chained to the previous + run of the same task, or each task could keep a head, so that a deleted run + can be detected. This also closes a v1 gap. When a task is marked done after + a run that held a task-bound grant has already settled, that run's file has + no `grant.revoked` line. The sample shows this with Ana's grant. +- **Live revocation.** A brokered credential, behind a proxy rather than in an + environment variable, would let `grant.revoked` take effect inside a running + process. +- **Pod-level views.** These are a filter on `podId`, which v1 already writes + on every task-bound line. + +## 7. Sample + +`examples/` is generated by `test/record-example.test.js`. A test fails if the +files drift from what the code writes. + +- `run-ana-0001.jsonl`: Ana's agent (opencode 1.19.0) claims `TASK-101` in + `pod-checkout`. + - Its grant attenuates Ben's Sentry connection grant. + - The first attempt is stuck, and the agent is retried. + - A `git push` is denied by the policy. + - A Sentry write asks for approval, and Ben approves it as + `credential-owner` through a decision card. + - The run reports a PR. +- `run-ben-0001.jsonl`: Ben's agent (goose 1.54.0) reviews the PR under a + read grant bound to the task. The board then marks the task done, which + revokes that grant in the same run. +- `TASK-101.otlp.json`: both runs exported as one trace. diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index 3ac1f5ce1f..9592997438 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -10,3 +10,7 @@ export { createTraceStore } from './trace/store.js'; export { validateGrant, matchToolCall, TOOL_KINDS, CREDENTIAL_SOURCES } from './valet/grant.js'; export { prepareRunEnv, BASE_ENV_ALLOWLIST } from './valet/env.js'; export { createPermissionPolicy, DEFAULT_APPROVE_TIMEOUT_MS } from './valet/permissions.js'; +export { + RECORD_VERSION, RECORD_SCHEMA, KINDS, canonicalLine, appendEntry, verifyRecord, readRecord, hashLine, toolSurfaceOf, +} from './record/format.js'; +export { toOtlpJson } from './record/otel.js'; diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js new file mode 100644 index 0000000000..406422e974 --- /dev/null +++ b/packages/seats/src/record/format.js @@ -0,0 +1,451 @@ +/** + * The run record, format v1: one canonical JSON object per line, each line + * hash-chained to the one before it. docs/record-v1.md is the specification; + * this file is its reference writer and verifier, and nothing here imports + * the store, so another tool can vendor it alone. + */ +import { createHash } from 'node:crypto'; +import { FAILURE_KINDS } from '../failures.js'; + +export const RECORD_VERSION = 1; +export const RECORD_SCHEMA = 'seats.record'; + +export const ACTOR_TYPES = Object.freeze(['agent', 'human', 'policy', 'system']); +// Who decides an approval today is the owner of the credential it would use; +// `decision-owner` is reserved for a separately assigned decider. +export const DECIDED_BY_ROLES = Object.freeze(['credential-owner', 'decision-owner', 'agent-owner', 'policy']); +export const TOOL_SURFACES = Object.freeze(['connector', 'commonly', 'native']); +export const GRANT_ENDS_ON = Object.freeze(['task.done', 'expiresAt']); +export const REVOKE_REASONS = Object.freeze(['task.done', 'expired', 'cascade', 'manual']); +export const WRITE_MODES = Object.freeze(['read', 'write', 'write-with-confirm']); +export const GRANT_TARGET_KINDS = Object.freeze(['pod', 'seat']); +export const OUTCOMES = Object.freeze(['ok', 'failed', 'stuck', 'cancelled']); +export const RESULT_STATUSES = Object.freeze(['done', 'partial', 'blocked', 'failed']); + +/** The actors a writer records for its own bookkeeping, and for the board. */ +export const SYSTEM_ACTOR = Object.freeze({ type: 'system', id: 'seats' }); +export const BOARD_ACTOR = Object.freeze({ type: 'system', id: 'board' }); + +/** + * Every v1 kind. `actor` is who a writer records when the emitter names + * nobody: 'run' (the run's own actor), 'system', or derived from the data + * ('issuer', 'decider'). `trace` is the EVENT_TYPES name, when it differs. + */ +export const KINDS = Object.freeze({ + 'task.claimed': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.released': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.done': Object.freeze({ group: 'claim', actor: 'board' }), + 'grant.issued': Object.freeze({ group: 'grant', actor: 'issuer' }), + 'grant.revoked': Object.freeze({ group: 'grant', actor: 'system' }), + 'permission.requested': Object.freeze({ group: 'approval', actor: 'run' }), + 'permission.decided': Object.freeze({ group: 'approval', actor: 'decider' }), + 'attempt.started': Object.freeze({ group: 'attempt', actor: 'run', trace: 'run.started' }), + 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + 'tool.call': Object.freeze({ group: 'tool', actor: 'run' }), + 'tool.update': Object.freeze({ group: 'tool', actor: 'run' }), + 'credential.granted': Object.freeze({ group: 'credential', actor: 'system' }), + 'credential.denied': Object.freeze({ group: 'credential', actor: 'system' }), + 'result.reported': Object.freeze({ group: 'result', actor: 'run' }), + 'turn.prompt_sent': Object.freeze({ group: 'activity', actor: 'system' }), + 'agent.message_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.thought_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + plan: Object.freeze({ group: 'activity', actor: 'run' }), + usage: Object.freeze({ group: 'activity', actor: 'run' }), + 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), + 'liveness.recovery': Object.freeze({ group: 'activity', actor: 'system' }), + 'process.exited': Object.freeze({ group: 'activity', actor: 'system' }), +}); + +const TRACE_TO_KIND = new Map(Object.entries(KINDS).filter(([, spec]) => spec.trace).map(([kind, spec]) => [spec.trace, kind])); + +/** Record kind for an EVENT_TYPES name or a kind; null when it is neither. */ +export const kindOfType = (type) => TRACE_TO_KIND.get(type) || (Object.hasOwn(KINDS, type) ? type : null); +/** The EVENT_TYPES name readRun reports for a kind (unknown kinds pass through). */ +export const traceTypeOf = (kind) => KINDS[kind]?.trace || kind; + +const fail = (message) => { throw new Error(`record: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isText = (value) => typeof value === 'string' && value.length > 0; +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/; +const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; +// A user id is an opaque token from the instance: no spaces, so a display +// name can't pass for one. Display names go in actor.name. +const USER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +export const isUserId = (value) => typeof value === 'string' && USER_ID.test(value); +const TOOL_NAME = /^[^\s*?]{1,256}$/; +const RFC3339_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +const KIND_SHAPE = /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$/; +const isTimestamp = (value) => typeof value === 'string' && RFC3339_UTC.test(value) && !Number.isNaN(Date.parse(value)); + +/** + * RFC 8785 (JCS) serialization. For the JSON values a record holds, JCS is + * JSON.stringify with object keys sorted by UTF-16 code units, which is what + * Array.prototype.sort does by default. + */ +const canonical = (value) => { + if (value !== null && typeof value === 'object' && typeof value.toJSON === 'function') return canonical(value.toJSON()); + if (value === null) return 'null'; + switch (typeof value) { + case 'string': return JSON.stringify(value); + case 'boolean': return value ? 'true' : 'false'; + case 'number': + if (!Number.isFinite(value)) fail('numbers must be finite'); + return JSON.stringify(value); + case 'object': + if (Array.isArray(value)) return `[${value.map((item) => (item === undefined ? 'null' : canonical(item))).join(',')}]`; + return `{${Object.keys(value).filter((key) => value[key] !== undefined).sort() + .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + default: return fail(`cannot serialize a ${typeof value}`); + } +}; + +/** The canonical line for an entry, without its trailing newline. */ +export const canonicalLine = (entry) => { + if (!isPlainObject(entry)) fail('an entry is a JSON object'); + return canonical(entry); +}; + +/** Lowercase hex SHA-256 over a line's UTF-8 bytes: what the next `prev` holds. */ +export const hashLine = (line) => createHash('sha256').update(String(line), 'utf8').digest('hex'); + +/** null when the actor is well formed, else the reason. */ +const actorProblem = (actor) => { + if (actor === null) return null; + if (!isPlainObject(actor)) return 'actor must be an object or null'; + if (!ACTOR_TYPES.includes(actor.type)) return `actor.type must be one of ${ACTOR_TYPES.join(', ')}`; + if (!isUserId(actor.id)) return 'actor.id must be an id token (a user id for agent and human actors), never a display name'; + if (actor.owner !== undefined && (actor.type !== 'agent' || !isUserId(actor.owner))) return 'actor.owner is a user id, on agent actors only'; + if (actor.name !== undefined && !isText(actor.name)) return 'actor.name must be a non-empty string'; + return null; +}; + +export const validateActor = (actor) => { + const problem = actorProblem(actor); + if (problem) fail(problem); + if (actor === null) return null; + return Object.freeze({ + type: actor.type, id: actor.id, ...(actor.owner ? { owner: actor.owner } : {}), ...(actor.name ? { name: actor.name } : {}), + }); +}; + +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); +}; +const envNames = (names, where) => { + if (!Array.isArray(names) || !names.every((name) => typeof name === 'string' && ENV_NAME.test(name))) { + fail(`${where} must be credential NAMES ([A-Z_][A-Z0-9_]*), never values`); + } + return [...names]; +}; +const optionalText = (value, where) => { + if (value !== undefined && value !== null && !isText(value)) fail(`${where} must be a non-empty string`); +}; +const userIdOrPolicy = (value, where) => { + if (value !== undefined && value !== null && value !== 'policy' && !isUserId(value)) fail(`${where} must be a user id or "policy"`); +}; + +/** A grant's scope, in the shape of a room grant: exact tool names and a connection id. */ +const checkScope = (scope) => { + if (!isPlainObject(scope)) fail('grant.issued needs a scope'); + refuseUnknownKeys(scope, ['connectionId', 'tools', 'writeMode', 'budget', 'audience'], 'grant.issued.scope.'); + optionalText(scope.connectionId, 'grant.issued.scope.connectionId'); + const tools = scope.tools ?? []; + if (!Array.isArray(tools) || !tools.every((name) => typeof name === 'string' && TOOL_NAME.test(name))) { + fail('grant.issued.scope.tools must be exact tool names (no spaces, no * or ?), not patterns'); + } + if (scope.writeMode !== undefined && !WRITE_MODES.includes(scope.writeMode)) fail(`grant.issued.scope.writeMode must be one of ${WRITE_MODES.join(', ')}`); + let budget; + if (scope.budget !== undefined) { + if (!isPlainObject(scope.budget)) fail('grant.issued.scope.budget must be an object'); + refuseUnknownKeys(scope.budget, ['calls', 'windowMs'], 'grant.issued.scope.budget.'); + if (scope.budget.calls !== undefined && !(Number.isInteger(scope.budget.calls) && scope.budget.calls >= 0)) fail('grant.issued.scope.budget.calls must be an integer >= 0'); + if (scope.budget.windowMs !== undefined && !(Number.isInteger(scope.budget.windowMs) && scope.budget.windowMs >= 1)) fail('grant.issued.scope.budget.windowMs must be an integer >= 1'); + budget = { ...scope.budget }; + } + const audience = scope.audience ?? []; + if (!Array.isArray(audience) || !audience.every(isUserId)) fail('grant.issued.scope.audience must be user ids'); + return { + ...(scope.connectionId ? { connectionId: scope.connectionId } : {}), + tools: [...tools], + ...(scope.writeMode ? { writeMode: scope.writeMode } : {}), + ...(budget ? { budget } : {}), + audience: [...audience], + }; +}; + +// Per-kind checks, applied by every writer. They are strictest where a value +// could ride in: a grant's scope, target and binding refuse fields they don't define. +const CHECKS = { + 'grant.issued': (data, envelope) => { + refuseUnknownKeys(data, ['grantId', 'parentGrantId', 'scope', 'target', 'issuedBy', 'expiresAt', 'binding'], 'grant.issued.'); + if (!isText(data.grantId)) fail('grant.issued needs a grantId'); + optionalText(data.parentGrantId, 'grant.issued.parentGrantId'); + const scope = checkScope(data.scope); + let target; + if (data.target !== undefined) { + if (!isPlainObject(data.target)) fail('grant.issued.target must be an object'); + refuseUnknownKeys(data.target, ['kind', 'id'], 'grant.issued.target.'); + if (!GRANT_TARGET_KINDS.includes(data.target.kind) || !isText(data.target.id)) fail('grant.issued.target is {kind: pod|seat, id}'); + target = { kind: data.target.kind, id: data.target.id }; + } + userIdOrPolicy(data.issuedBy, 'grant.issued.issuedBy'); + if (data.expiresAt !== undefined && !isTimestamp(data.expiresAt)) fail('grant.issued.expiresAt must be an RFC 3339 UTC timestamp'); + let binding; + if (data.binding !== undefined) { + if (!isPlainObject(data.binding)) fail('grant.issued.binding must be an object'); + refuseUnknownKeys(data.binding, ['taskId', 'podId', 'endsOn'], 'grant.issued.binding.'); + if (!isText(data.binding.taskId) || !isText(data.binding.podId)) fail('grant.issued.binding needs taskId and podId: task ids repeat across pods'); + if (data.binding.taskId !== envelope.taskId || data.binding.podId !== envelope.podId) fail('grant.issued.binding must name the run\'s own taskId and podId'); + if (!GRANT_ENDS_ON.includes(data.binding.endsOn)) fail(`grant.issued.binding.endsOn must be one of ${GRANT_ENDS_ON.join(', ')}`); + if (data.binding.endsOn === 'expiresAt' && data.expiresAt === undefined) fail('a grant that ends on expiresAt needs an expiresAt'); + binding = { taskId: data.binding.taskId, podId: data.binding.podId, endsOn: data.binding.endsOn }; + } + return { + ...data, + ...(data.parentGrantId === undefined ? {} : { parentGrantId: data.parentGrantId }), + scope, + ...(target ? { target } : {}), + ...(binding ? { binding } : {}), + }; + }, + 'grant.revoked': (data) => { + if (!Array.isArray(data.grantIds) || data.grantIds.length === 0 || !data.grantIds.every(isText)) fail('grant.revoked needs grantIds: a non-empty array'); + if (new Set(data.grantIds).size !== data.grantIds.length) fail('grant.revoked.grantIds repeats an id'); + if (!REVOKE_REASONS.includes(data.reason)) fail(`grant.revoked.reason must be one of ${REVOKE_REASONS.join(', ')}`); + return { ...data, grantIds: [...data.grantIds] }; + }, + 'task.done': (data, envelope) => { + if (!envelope.taskId) fail('task.done needs a run with a taskId'); + userIdOrPolicy(data.doneBy, 'task.done.doneBy'); + return data; + }, + 'permission.decided': (data) => { + // A rule-decided call never had a person in the loop; a writer says so + // rather than leave a reader to infer it from rule.list. + const out = { ...data }; + if (!('decidedBy' in out)) { + const byRule = out.rule && out.rule.list !== 'ask'; + out.decidedBy = byRule ? 'policy' : null; + if (!('decidedByRole' in out)) out.decidedByRole = byRule ? 'policy' : null; + } + if (!('decidedByRole' in out)) out.decidedByRole = null; + if (out.decidedBy !== null) userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); + if (out.decidedByRole !== null && !DECIDED_BY_ROLES.includes(out.decidedByRole)) { + fail(`permission.decided.decidedByRole must be one of ${DECIDED_BY_ROLES.join(', ')}, or null`); + } + optionalText(out.decisionCardId, 'permission.decided.decisionCardId'); + return out; + }, + 'tool.call': (data) => { + if (data.surface !== undefined && !TOOL_SURFACES.includes(data.surface)) fail(`tool.call.surface must be one of ${TOOL_SURFACES.join(', ')}`); + return data; + }, + 'credential.granted': (data) => ({ ...data, names: envNames(data.names ?? [], 'credential.granted.names') }), + 'credential.denied': (data) => { + envNames([data.name], 'credential.denied.name'); + return data; + }, + 'attempt.settled': (data) => { + if (!Number.isInteger(data.attempt) || data.attempt < 1) fail('attempt.settled.attempt must be a positive integer'); + if (!OUTCOMES.includes(data.outcome)) fail(`attempt.settled.outcome must be one of ${OUTCOMES.join(', ')}`); + if (data.failure !== undefined && data.failure !== null) { + const { kind } = data.failure; + if (kind !== null && kind !== undefined && !FAILURE_KINDS[kind]) fail(`attempt.settled.failure.kind ${kind} is not a FAILURE_KINDS key`); + } + return data; + }, + 'result.reported': (data) => { + if (!RESULT_STATUSES.includes(data.status)) fail(`result.reported.status must be one of ${RESULT_STATUSES.join(', ')}`); + if (data.refs !== undefined && !(Array.isArray(data.refs) && data.refs.every((ref) => isPlainObject(ref) && isText(ref.type) && isText(ref.id)))) { + fail('result.reported.refs must be [{type, id}]'); + } + return data; + }, +}; + +/** + * Validate one kind's data and return the copy a writer records. `envelope` + * is {taskId, podId}, for checks that tie data to the run it sits in. + */ +export const normalizeData = (kind, data = {}, envelope = {}) => { + if (!isPlainObject(data)) fail(`${kind} data must be an object`); + const check = CHECKS[kind]; + return check ? check({ ...data }, envelope) : { ...data }; +}; + +/** A grant's rules acting on their own: named by the grant when its id is a token. */ +const policyActor = (grantId) => ({ type: 'policy', id: isUserId(`grant:${grantId}`) ? `grant:${grantId}` : 'policy' }); + +/** The actor a writer records when the emitter names none. */ +export const defaultActor = (kind, data = {}, runActor = null) => { + switch (KINDS[kind]?.actor) { + case 'run': return runActor; + case 'system': return SYSTEM_ACTOR; + case 'board': return BOARD_ACTOR; + case 'issuer': + if (data.issuedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.issuedBy) ? { type: 'human', id: data.issuedBy } : null; + case 'decider': + if (data.decidedByRole === 'policy' || data.decidedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.decidedBy) ? { type: 'human', id: data.decidedBy } : null; + default: return null; + } +}; + +const parseLine = (line) => { + let entry; + try { entry = JSON.parse(line); } catch { return fail('the previous line is not JSON'); } + if (!isPlainObject(entry) || !Number.isInteger(entry.seq)) fail('the previous line is not a record entry'); + return entry; +}; + +/** + * appendEntry(prevLine, fields, { runActor }?) → the next line, canonical, + * without its newline. + * + * fields: { runId, taskId?, podId?, ts, kind, actor?, data, seq? }. seq and + * prev come from prevLine (null for the first line); a given seq must agree. + * An absent actor takes the kind's default; null records "unknown". + */ +export const appendEntry = (prevLine, fields = {}, { runActor = null } = {}) => { + let seq = 1; + let prev = null; + if (prevLine !== null && prevLine !== undefined) { + const before = parseLine(prevLine); + seq = before.seq + 1; + prev = hashLine(prevLine); + if (before.runId !== fields.runId) fail('one record holds one run'); + if (before.taskId !== fields.taskId || before.podId !== fields.podId) fail('taskId and podId are fixed for the whole run'); + if (before.kind === 'run.settled') fail('nothing is written after run.settled'); + } + if (fields.seq !== undefined && fields.seq !== seq) fail(`seq ${fields.seq} does not follow ${seq - 1}`); + if (!RUN_ID.test(String(fields.runId))) fail('runId must be 6-64 characters of [A-Za-z0-9_-]'); + optionalText(fields.taskId, 'taskId'); + optionalText(fields.podId, 'podId'); + if (fields.taskId && !fields.podId) fail('a taskId needs a podId: task ids repeat across pods'); + const ts = fields.ts instanceof Date ? fields.ts.toISOString() : fields.ts; + if (!isTimestamp(ts)) fail('ts must be an RFC 3339 UTC timestamp'); + if (!Object.hasOwn(KINDS, fields.kind)) fail(`unknown record kind: ${fields.kind}`); + const data = normalizeData(fields.kind, fields.data ?? {}, { taskId: fields.taskId, podId: fields.podId }); + const actor = validateActor(fields.actor === undefined ? defaultActor(fields.kind, data, runActor) : fields.actor); + return canonicalLine({ + v: RECORD_VERSION, + schema: RECORD_SCHEMA, + runId: fields.runId, + ...(fields.taskId ? { taskId: fields.taskId } : {}), + ...(fields.podId ? { podId: fields.podId } : {}), + seq, + ts, + kind: fields.kind, + actor, + data, + prev, + }); +}; + +/** Split a record file's text into lines; a last line with no newline is torn. */ +export const splitRecord = (text) => { + if (!text) return { lines: [], torn: null }; + const parts = String(text).split('\n'); + const tail = parts.pop(); + return { lines: parts, torn: tail === '' ? null : tail }; +}; + +const envelopeProblem = (entry) => { + if (!Number.isInteger(entry.v) || entry.v < 1) return 'v'; + if (entry.schema !== RECORD_SCHEMA) return 'schema'; + if (!RUN_ID.test(String(entry.runId))) return 'runId'; + if (entry.taskId !== undefined && !isText(entry.taskId)) return 'taskId'; + if (entry.podId !== undefined && !isText(entry.podId)) return 'podId'; + if (entry.taskId !== undefined && entry.podId === undefined) return 'podId'; + if (!Number.isInteger(entry.seq) || entry.seq < 1) return 'seq'; + if (!isTimestamp(entry.ts)) return 'ts'; + if (typeof entry.kind !== 'string' || !KIND_SHAPE.test(entry.kind)) return 'kind'; + if (actorProblem(entry.actor)) return 'actor'; + if (!isPlainObject(entry.data)) return 'data'; + if (entry.prev !== null && !(typeof entry.prev === 'string' && /^[0-9a-f]{64}$/.test(entry.prev))) return 'prev'; + return null; +}; + +/** + * verifyRecord(text | lines) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } + * + * Text is a whole file, so a torn final line (no newline) can be seen; an + * array is taken as complete lines. Each problem is reported at the seq of + * the line where it was observed: + * torn the last line was cut short by a crash mid-write + * unparseable not JSON + * bad_envelope an envelope field is missing or malformed (detail names it) + * unsupported_version v is newer than this reader + * not_canonical the bytes are not the canonical form of what they parse to + * seq_mismatch seq is not the previous seq + 1 (a deleted or moved line) + * prev_mismatch prev is not the hash of the line before it: that line, or + * this one's prev, was changed, removed or moved + * run_mismatch runId, taskId or podId differs from the first line + * after_settled a line follows run.settled + * The chain re-syncs after each problem, so one edit is reported once. + */ +export const verifyRecord = (input) => { + const { lines, torn } = Array.isArray(input) ? { lines: input.map(String), torn: null } : splitRecord(input); + const errors = []; + const report = (seq, reason, detail) => errors.push(detail ? { seq, reason, detail } : { seq, reason }); + let expected = 1; + let prevRaw = null; + let first = null; + let settled = false; + for (const line of lines) { + let entry; + try { entry = JSON.parse(line); } catch { entry = undefined; } + if (!isPlainObject(entry)) { + report(expected, 'unparseable'); + prevRaw = line; + expected += 1; + continue; + } + const seq = Number.isInteger(entry.seq) ? entry.seq : expected; + if (Number.isInteger(entry.v) && entry.v > RECORD_VERSION) report(seq, 'unsupported_version', `v${entry.v}`); + const problem = envelopeProblem(entry); + if (problem) report(seq, 'bad_envelope', problem); + let canonicalForm = null; + try { canonicalForm = canonicalLine(entry); } catch { /* reported below */ } + if (canonicalForm !== line) report(seq, 'not_canonical'); + if (seq !== expected) report(seq, 'seq_mismatch', `expected ${expected}`); + const expectedPrev = prevRaw === null ? null : hashLine(prevRaw); + if (entry.prev !== expectedPrev) report(seq, 'prev_mismatch'); + if (first === null) first = entry; + else if (entry.runId !== first.runId || entry.taskId !== first.taskId || entry.podId !== first.podId) report(seq, 'run_mismatch'); + if (settled) report(seq, 'after_settled'); + if (entry.kind === 'run.settled') settled = true; + prevRaw = line; + expected = seq + 1; + } + if (torn !== null) report(expected, 'torn'); + return { + ok: errors.length === 0, + errors, + entries: lines.length, + head: prevRaw === null ? null : hashLine(prevRaw), + settled, + }; +}; + +/** Parse a record's text into entries, skipping a torn tail and unparseable lines. */ +export const readRecord = (text) => splitRecord(text).lines.flatMap((line) => { + try { + const entry = JSON.parse(line); + return isPlainObject(entry) ? [entry] : []; + } catch { return []; } +}); + +/** + * Which surface a tool name belongs to, for an adapter that knows the real + * tool name (an MCP server/tool pair, say). Never apply it to a title: a + * title is the agent's own text and can claim any prefix. + */ +export const toolSurfaceOf = (toolName) => { + const name = String(toolName || ''); + if (/^(?:mcp__commonly__|commonly_)/.test(name)) return 'commonly'; + if (/^mcp__[^_]/.test(name)) return 'connector'; + return 'native'; +}; diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js new file mode 100644 index 0000000000..a65acf1a77 --- /dev/null +++ b/packages/seats/src/record/otel.js @@ -0,0 +1,214 @@ +/** + * Export run records as OTLP/JSON traces (an ExportTraceServiceRequest), so + * a tracing backend can show who let which agent do what beside its own + * spans. docs/record-v1.md §5 is the mapping; the gen_ai.* names follow the + * OpenTelemetry GenAI conventions, which are Development status upstream. + * + * Ids are derived, never random: the same entries export to the same bytes. + */ +import { createHash } from 'node:crypto'; +import { KINDS, RECORD_SCHEMA, RECORD_VERSION } from './format.js'; + +const SPAN_KIND_INTERNAL = 1; +const STATUS_UNSET = 0; +const STATUS_ERROR = 2; +const FAILED_OUTCOMES = new Set(['failed', 'stuck', 'cancelled']); +// Content stays out of the export, as upstream makes content capture opt-in. +const CONTENT_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']); + +const digest = (text, bytes) => { + const hex = createHash('sha256').update(text, 'utf8').digest('hex').slice(0, bytes * 2); + // An all-zero id is invalid in OTLP; vanishingly unlikely, cheap to exclude. + return /^0+$/.test(hex) ? digest(`${text}#`, bytes) : hex; +}; + +/** 16-byte trace id per task (task ids repeat across pods, so the pod is in it), else per run. */ +export const traceIdOf = (entry) => digest(entry.taskId + ? `seats.record/trace/task/${entry.podId}/${entry.taskId}` + : `seats.record/trace/run/${entry.runId}`, 16); +/** 8-byte span id for the span an entry opens; the run span uses seq 0. */ +export const spanIdOf = (runId, seq) => digest(`seats.record/span/${runId}/${seq}`, 8); + +/** RFC 3339 → Unix nanoseconds as a decimal string, keeping sub-millisecond digits. */ +const nanos = (ts) => { + const match = /^(.*T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z$/.exec(ts); + if (!match) return '0'; + return String(BigInt(Date.parse(`${match[1]}Z`)) * 1000000n + BigInt((match[2] || '').padEnd(9, '0'))); +}; + +const snake = (key) => key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); + +const anyValue = (value) => { + if (typeof value === 'string') return { stringValue: value }; + if (typeof value === 'boolean') return { boolValue: value }; + if (Number.isInteger(value)) return { intValue: String(value) }; + if (typeof value === 'number') return { doubleValue: value }; + if (Array.isArray(value)) return { arrayValue: { values: value.filter((v) => v !== null && v !== undefined).map(anyValue) } }; + return { + kvlistValue: { + values: Object.keys(value).filter((k) => value[k] !== null && value[k] !== undefined).sort() + .map((k) => ({ key: snake(k), value: anyValue(value[k]) })), + }, + }; +}; +const attr = (key, value) => ({ key, value: anyValue(value) }); +const attrs = (pairs) => pairs.filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => attr(key, value)); + +/** Nested data → `seats.<path>` attributes, keys sorted for a stable export. */ +const flatten = (data, prefix = 'seats') => Object.keys(data || {}).sort().flatMap((key) => { + const value = data[key]; + const name = `${prefix}.${snake(key)}`; + if (value === null || value === undefined) return []; + if (typeof value === 'object' && !Array.isArray(value)) return flatten(value, name); + return [attr(name, value)]; +}); + +const eventOf = (entry) => ({ + timeUnixNano: nanos(entry.ts), + name: entry.kind, + attributes: [ + attr('seats.seq', entry.seq), + ...(entry.actor ? attrs([ + ['seats.actor.type', entry.actor.type], ['seats.actor.id', entry.actor.id], + ['seats.actor.owner', entry.actor.owner], ['seats.actor.name', entry.actor.name], + ]) : []), + ...flatten(entry.data), + ], +}); + +const fail = (span, errorType) => { + span.status = { code: STATUS_ERROR, message: errorType }; + span.attributes.push(attr('error.type', errorType)); +}; + +/** One run's entries, in seq order → its spans. */ +const spansOfRun = (entries) => { + const head = entries[0]; + const traceId = traceIdOf(head); + const last = entries[entries.length - 1]; + const span = ({ seq, parent, name, attributes, start }) => ({ + traceId, + spanId: spanIdOf(head.runId, seq), + ...(parent ? { parentSpanId: parent.spanId } : {}), + name, + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: nanos(start), + endTimeUnixNano: null, + attributes, + events: [], + status: { code: STATUS_UNSET }, + }); + const run = span({ + seq: 0, + name: 'seats.run', + start: head.ts, + attributes: attrs([['seats.run.id', head.runId], ['seats.task.id', head.taskId], ['seats.pod.id', head.podId]]), + }); + const spans = [run]; + let attempt = null; // { span, number, tools: Map<toolCallId, span>, usage } + const end = (s, ts) => { if (s && s.endTimeUnixNano === null) s.endTimeUnixNano = nanos(ts); }; + const closeAttempt = (ts) => { + if (!attempt) return; + for (const tool of attempt.tools.values()) end(tool, ts); + if (attempt.usage) { + attempt.span.attributes.push(...attrs([ + ['gen_ai.usage.input_tokens', attempt.usage.inputTokens], ['gen_ai.usage.output_tokens', attempt.usage.outputTokens], + ])); + } + end(attempt.span, ts); + attempt = null; + }; + const current = () => attempt?.span || run; + + for (const entry of entries) { + const { kind, data = {} } = entry; + if (!Object.hasOwn(KINDS, kind) || CONTENT_KINDS.has(kind)) continue; // unknown kinds are skipped + if (kind === 'attempt.started') { + closeAttempt(entry.ts); + const agentName = entry.actor?.name || entry.actor?.id; + const s = span({ + seq: entry.seq, + parent: run, + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'invoke_agent'], + ['gen_ai.agent.id', entry.actor?.id], ['gen_ai.agent.name', agentName], + ['gen_ai.conversation.id', data.sessionId], + ['seats.agent.owner', entry.actor?.owner], + ['seats.attempt.number', data.attempt], ['seats.adapter', data.adapter], + ['seats.cli.name', data.cli?.name], ['seats.cli.version', data.cli?.version], + ]), + }); + spans.push(s); + attempt = { span: s, number: data.attempt, agentName, tools: new Map(), usage: null }; + } else if (kind === 'attempt.settled' && attempt && attempt.number === data.attempt) { + attempt.span.attributes.push(attr('seats.attempt.outcome', data.outcome)); + if (FAILED_OUTCOMES.has(data.outcome)) fail(attempt.span, data.failure?.kind || data.outcome); + closeAttempt(entry.ts); + } else if (kind === 'run.settled') { + run.attributes.push(...attrs([['seats.run.outcome', data.outcome], ['seats.run.attempts', data.attempts]])); + if (FAILED_OUTCOMES.has(data.outcome)) { + const errorType = data.failure?.kind || data.outcome; + fail(run, errorType); + // The run's outcome is the last attempt's when nothing settled it. + if (attempt) fail(attempt.span, errorType); + } + closeAttempt(entry.ts); + end(run, entry.ts); + } else if (kind === 'tool.call') { + const toolName = data.name || data.title || 'tool'; + const s = span({ + seq: entry.seq, + parent: current(), + name: `execute_tool ${toolName}`, + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'execute_tool'], + ['gen_ai.tool.name', toolName], ['gen_ai.tool.call.id', data.toolCallId], + ['gen_ai.agent.name', attempt?.agentName], + ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], + ]), + }); + spans.push(s); + if (attempt && data.toolCallId) attempt.tools.set(data.toolCallId, s); + } else if (kind === 'tool.update') { + const tool = attempt?.tools.get(data.toolCallId); + (tool || current()).events.push(eventOf(entry)); + if (tool && data.status === 'failed') fail(tool, 'tool_failed'); + if (tool && (data.status === 'completed' || data.status === 'failed')) end(tool, entry.ts); + } else if (kind === 'usage' && attempt) { + attempt.usage = data; + attempt.span.events.push(eventOf(entry)); + } else if (kind === 'permission.requested' || kind === 'permission.decided') { + (attempt?.tools.get(data.toolCallId) || current()).events.push(eventOf(entry)); + } else { + current().events.push(eventOf(entry)); + } + } + closeAttempt(last.ts); + end(run, last.ts); + return spans; +}; + +/** + * toOtlpJson(entries, { serviceName? }) → ExportTraceServiceRequest (OTLP/JSON) + * + * `entries` are record entries (readRun's output, or parsed lines) from one + * or more runs. Entries of another schema or a newer version are left out. + */ +export const toOtlpJson = (entries, { serviceName = 'seats' } = {}) => { + const runs = new Map(); + for (const entry of entries || []) { + if (!entry || entry.schema !== RECORD_SCHEMA || !(entry.v <= RECORD_VERSION)) continue; + if (!runs.has(entry.runId)) runs.set(entry.runId, []); + runs.get(entry.runId).push(entry); + } + const spans = [...runs.values()].flatMap((run) => spansOfRun([...run].sort((a, b) => a.seq - b.seq))); + return { + resourceSpans: [{ + resource: { attributes: [attr('service.name', serviceName)] }, + scopeSpans: [{ scope: { name: RECORD_SCHEMA, version: String(RECORD_VERSION) }, spans }], + }], + }; +}; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index a6a02388a4..6f6d0a8fb3 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -2,8 +2,10 @@ * Trace event vocabulary. Names follow ACP's session/update kinds where one * exists, so an ACP agent's stream maps across without a translation table. * - * Event: { runId, seq, ts, type, data }. `seq` is per run, starting at 1; - * `ts` is an ISO string. Events never carry prompt text, credentials or + * Event: { runId, seq, ts, type, data }, plus the record envelope (kind, v, + * actor, prev, taskId?, podId?) when read back from the store; the on-disk + * format is docs/record-v1.md. `seq` is per run, starting at 1; `ts` is + * an ISO string. Events never carry prompt text, credentials or * environment values — only names, ids, sizes and the agent's own output. */ export const EVENT_TYPES = Object.freeze([ @@ -23,6 +25,14 @@ export const EVENT_TYPES = Object.freeze([ 'credential.granted', // { names: [...] } names only, never values 'credential.denied', // { name, reason?: 'not_granted'|'unresolved' } 'process.exited', // { code, signal } + // Record kinds with no earlier trace name (docs/record-v1.md has the full data). + 'task.claimed', // { leaseUntil? } mirrors the board's claim; the board stays the source of truth + 'task.released', // { reason? } + 'task.done', // { doneBy? } written for the board; ends grants bound to the task + 'grant.issued', // { grantId, parentGrantId?, scope: {connectionId?, tools: [names], writeMode?, budget?, audience}, target?, issuedBy?, expiresAt?, binding? } + 'grant.revoked', // { grantIds: [...], reason: 'task.done'|'expired'|'cascade'|'manual' } + 'attempt.settled', // { attempt, outcome, failure?: {kind, detail} } + 'result.reported', // { status: 'done'|'partial'|'blocked'|'failed', refs?: [{type, id}] } ]); const KNOWN = new Set(EVENT_TYPES); diff --git a/packages/seats/src/trace/store.js b/packages/seats/src/trace/store.js index bacff1e8b1..4608844385 100644 --- a/packages/seats/src/trace/store.js +++ b/packages/seats/src/trace/store.js @@ -1,15 +1,20 @@ /** * Append-only trace store on the local filesystem. * - * <root>/runs/<runId>.jsonl one event per line, in seq order - * <root>/index.jsonl one line per started run, one per settled run + * <root>/runs/<runId>.jsonl the run record (docs/record-v1.md): one + * canonical, hash-chained entry per line + * <root>/index.jsonl one line per started run, one per settled run; + * a store convenience, not part of the format * * Files are 0600 and directories 0700: a trace holds agent output. */ import { randomUUID } from 'node:crypto'; import { appendFileSync, mkdirSync, readFileSync, existsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import { isEventType } from './events.js'; +import { + appendEntry, hashLine, kindOfType, readRecord, traceTypeOf, validateActor, verifyRecord, BOARD_ACTOR, +} from '../record/format.js'; +import { redactKnownCredentials } from '../upstream-refusal.js'; const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; const readJsonl = (path) => (existsSync(path) @@ -18,6 +23,20 @@ const readJsonl = (path) => (existsSync(path) }) : []); +/** Known credential values never reach a line, even inside agent text. */ +const scrub = (value, secrets) => { + if (secrets.length === 0) return value; + if (typeof value === 'string') return redactKnownCredentials(value, secrets); + if (Array.isArray(value)) return value.map((item) => scrub(item, secrets)); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrub(item, secrets)])); + } + return value; +}; + +/** What readRun has always returned, plus the envelope: `type` is the trace name. */ +const asEvent = (entry) => (entry.kind ? { ...entry, type: traceTypeOf(entry.kind) } : entry); + export const createTraceStore = ({ root, now = () => new Date() } = {}) => { if (!root) throw new Error('createTraceStore needs a root directory'); const runsDir = join(root, 'runs'); @@ -25,9 +44,21 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { const append = (path, value) => appendFileSync(path, `${JSON.stringify(value)}\n`, { mode: 0o600 }); const indexPath = join(root, 'index.jsonl'); + /** + * startRun({ runId?, adapter?, label?, taskId?, podId?, actor? }) + * → { runId, emit, redact, revokeGrant, expireGrants, openGrants, settled } + * + * `taskId` is the board's task id verbatim and needs `podId`, since task + * ids repeat across pods. `actor` is the run's own party, normally + * {type: 'agent', id: <user id>, owner: <user id>}. + */ const startRun = (meta = {}) => { const runId = meta.runId || randomUUID(); if (!RUN_ID.test(runId)) throw new Error('runId must be 6-64 characters of [A-Za-z0-9_-]'); + const taskId = meta.taskId || undefined; + const podId = meta.podId || undefined; + if (taskId && !podId) throw new Error('a taskId needs a podId: task ids repeat across pods'); + const runActor = meta.actor === undefined ? null : validateActor(meta.actor); const path = join(runsDir, `${runId}.jsonl`); try { writeFileSync(path, '', { mode: 0o600, flag: 'wx' }); @@ -35,23 +66,105 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { if (error.code === 'EEXIST') throw new Error(`run ${runId} already exists`); throw error; } + let last = null; // the previous line, which the next one hashes let seq = 0; let settled = false; + let broken = false; + const secrets = []; + const grants = new Map(); // open grants: grantId → { parentGrantId, binding, expiresAt } const startedAt = now(); - const emit = (type, data = {}) => { - if (!isEventType(type)) throw new Error(`unknown trace event type: ${type}`); - if (settled) return null; + + const write = (kind, data, actor) => { + const line = appendEntry(last, { + runId, taskId, podId, ts: now().toISOString(), kind, data: scrub(data, secrets), actor, + }, { runActor }); + try { + appendFileSync(path, `${line}\n`, { mode: 0o600 }); + } catch (error) { + // Some bytes may have landed; a chain cannot continue past an unknown tail. + broken = true; + throw error; + } + last = line; seq += 1; - const event = { runId, seq, ts: now().toISOString(), type, data }; - append(path, event); - if (type === 'run.settled') { + const entry = JSON.parse(line); + if (kind === 'grant.issued') { + grants.set(entry.data.grantId, { + parentGrantId: entry.data.parentGrantId ?? null, binding: entry.data.binding || null, expiresAt: entry.data.expiresAt || null, + }); + } + if (kind === 'grant.revoked') for (const id of entry.data.grantIds) grants.delete(id); + return asEvent(entry); + }; + + const revoke = (grantIds, reason, actor) => (grantIds.length ? write('grant.revoked', { grantIds, reason }, actor) : null); + + const emit = (type, data = {}, options = {}) => { + const kind = kindOfType(type); + if (!kind) throw new Error(`unknown trace event type: ${type}`); + if (settled) return null; + if (broken) throw new Error(`run ${runId}: the record is broken after a failed write`); + const event = write(kind, data, options?.actor); + if (kind === 'task.done') { + // The board closed the task: every grant bound to it ends, in one write. + const bound = [...grants].filter(([, g]) => g.binding?.endsOn === 'task.done').map(([id]) => id); + revoke(bound, 'task.done', BOARD_ACTOR); + } + if (kind === 'run.settled') { settled = true; - append(indexPath, { runId, settledAt: event.ts, outcome: data.outcome, failure: data.failure || null, durationMs: data.durationMs ?? null }); + append(indexPath, { + runId, + settledAt: event.ts, + outcome: event.data.outcome, + failure: event.data.failure || null, + durationMs: event.data.durationMs ?? null, + entries: seq, + head: hashLine(last), + }); } return event; }; - append(indexPath, { runId, startedAt: startedAt.toISOString(), adapter: meta.adapter || null, label: meta.label || null }); - return { runId, emit, get settled() { return settled; } }; + + /** Revoke a grant and every descendant this run knows of, in one entry. */ + const revokeGrant = (grantId, { actor } = {}) => { + if (settled || !grants.has(grantId)) return null; + const lineage = [grantId]; + for (let i = 0; i < lineage.length; i += 1) { + for (const [id, g] of grants) if (g.parentGrantId === lineage[i] && !lineage.includes(id)) lineage.push(id); + } + return revoke(lineage, lineage.length > 1 ? 'cascade' : 'manual', actor); + }; + + /** Record the end of every open grant whose expiresAt has passed. */ + const expireGrants = (at = now()) => { + if (settled) return null; + const cutoff = at.getTime(); + return revoke([...grants].filter(([, g]) => g.expiresAt && Date.parse(g.expiresAt) <= cutoff).map(([id]) => id), 'expired'); + }; + + const redact = (values = []) => { + for (const value of values) if (typeof value === 'string' && !secrets.includes(value)) secrets.push(value); + }; + + append(indexPath, { + runId, + startedAt: startedAt.toISOString(), + adapter: meta.adapter || null, + label: meta.label || null, + ...(taskId ? { taskId, podId } : podId ? { podId } : {}), + ...(runActor ? { actor: runActor } : {}), + }); + return { + runId, + taskId: taskId ?? null, + podId: podId ?? null, + emit, + redact, + revokeGrant, + expireGrants, + openGrants: () => [...grants.keys()], + get settled() { return settled; }, + }; }; const listRuns = ({ limit = 50 } = {}) => { @@ -64,11 +177,32 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { return [...byId.values()].sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt))).slice(0, limit); }; + const runPath = (runId) => (RUN_ID.test(String(runId)) ? join(runsDir, `${runId}.jsonl`) : null); + const readRun = (runId) => { - if (!RUN_ID.test(String(runId))) return null; - const path = join(runsDir, `${runId}.jsonl`); - return existsSync(path) ? readJsonl(path) : null; + const path = runPath(runId); + return path && existsSync(path) ? readRecord(readFileSync(path, 'utf8')).map(asEvent) : null; + }; + + /** + * verifyRun(runId) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } | null + * + * The chain check from verifyRecord, plus the last line against the head the + * index recorded at settle (head_mismatch, count_mismatch). The index sits + * beside the record, so that check catches accidents, not an editor. + */ + const verifyRun = (runId) => { + const path = runPath(runId); + if (!path || !existsSync(path)) return null; + const result = verifyRecord(readFileSync(path, 'utf8')); + const sealed = readJsonl(indexPath).filter((row) => row.runId === runId && row.head).pop(); + if (sealed) { + if (sealed.entries !== result.entries) result.errors.push({ seq: sealed.entries, reason: 'count_mismatch', detail: `index ${sealed.entries}, file ${result.entries}` }); + else if (sealed.head !== result.head) result.errors.push({ seq: sealed.entries, reason: 'head_mismatch' }); + result.ok = result.errors.length === 0; + } + return result; }; - return { root, startRun, listRuns, readRun }; + return { root, startRun, listRuns, readRun, verifyRun }; }; diff --git a/packages/seats/src/valet/env.js b/packages/seats/src/valet/env.js index ee38cdf50d..f819da8118 100644 --- a/packages/seats/src/valet/env.js +++ b/packages/seats/src/valet/env.js @@ -10,6 +10,7 @@ * This removes the paths a CLI discovers its login by; an OS sandbox * (Seatbelt, bwrap) is what stops a process that goes looking. */ +import { randomUUID } from 'node:crypto'; import { mkdtemp, mkdir, chmod, readFile, rm } from 'node:fs/promises'; import { homedir, tmpdir } from 'node:os'; import { delimiter, join, resolve, sep } from 'node:path'; @@ -75,22 +76,49 @@ const resolveOne = async (spec, { baseEnv, resolveSecret }) => { }; /** - * prepareRunEnv({ grant, baseEnv, resolveSecret, root?, require?, trace?, ctx? }) - * → { env, home, cleanup, granted: [names] } + * The grant.issued entry for grantMeta. Its scope is the room grant this run + * acts under (exact tool names, a connection id), as the host knows it; the + * run grant's own credential names and patterns show up in credential.* and + * in each permission.decided rule. `taskId` + `podId` bind the grant to that + * board task, ending when it is done unless `endsOn` says otherwise. + */ +const grantIssued = (meta) => { + const { + grantId = randomUUID(), parentGrantId, scope = {}, target, issuedBy, expiresAt, taskId, podId, endsOn = 'task.done', + } = meta; + return { + grantId, + ...(parentGrantId === undefined ? {} : { parentGrantId }), + scope: { tools: [], audience: [], ...scope }, + ...(target ? { target } : {}), + ...(issuedBy ? { issuedBy } : {}), + ...(expiresAt ? { expiresAt: expiresAt instanceof Date ? expiresAt.toISOString() : expiresAt } : {}), + ...(taskId ? { binding: { taskId, podId, endsOn } } : {}), + }; +}; + +/** + * prepareRunEnv({ grant, baseEnv, resolveSecret, root?, require?, trace?, ctx?, grantMeta? }) + * → { env, home, cleanup, granted: [names], grantId? } * * `require` lists credential names the adapter cannot run without; one the * grant does not name fails the run as credential_denied before any spawn. * A named credential that cannot be resolved fails it the same way: a run * that silently lacks its key fails later and less legibly. * - * Events go to `trace`, else ctx.trace: credential.granted {names} and - * credential.denied {name, reason}. Values never enter an event or a message. + * Events go to `trace`, else ctx.trace: grant.issued (only with grantMeta, + * first), then credential.granted {names} or credential.denied {name, reason}. + * Values never enter an event or a message, and a hook that offers + * redact(values) is handed the resolved values so the record can scrub them + * out of agent text too. */ export const prepareRunEnv = async ({ - grant, baseEnv = {}, resolveSecret, root, require: required = [], trace, ctx, + grant, baseEnv = {}, resolveSecret, root, require: required = [], trace, ctx, grantMeta, } = {}) => { const valid = validateGrant(grant); const hook = trace && typeof trace.emit === 'function' ? trace : traceOf(ctx); + const issued = grantMeta ? grantIssued(grantMeta) : null; + if (issued) hook.emit('grant.issued', issued); const realHomes = [...new Set([baseEnv.HOME, homedir()].filter(Boolean))]; const deny = (name, reason, values) => { hook.emit('credential.denied', { name, reason }); @@ -118,7 +146,8 @@ export const prepareRunEnv = async ({ await cleanup(); throw error; } + if (typeof hook.redact === 'function') hook.redact(values); const names = valid.credentials.map((c) => c.name); hook.emit('credential.granted', { names }); - return { env, home, cleanup, granted: names }; + return { env, home, cleanup, granted: names, ...(issued ? { grantId: issued.grantId } : {}) }; }; diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js index 6a2f609ad8..260a18372d 100644 --- a/packages/seats/src/valet/permissions.js +++ b/packages/seats/src/valet/permissions.js @@ -4,14 +4,18 @@ * * allow → the allow_once option * deny → the reject_once option, or cancelled when there is none - * ask → approve({ toolCall, grantRule }) from the host; true allows, - * anything else (false, a throw, no answer in time) denies + * ask → approve({ toolCall, grantRule }) from the host. `true` allows, + * and so does an object whose `approved` is exactly true: + * { approved: true, by: <user id>, role?, decisionCardId? }. + * Anything else (false, 'yes', {approved: 1}, a throw, no answer in + * time) denies. `by`/`role`/`decisionCardId` are recorded either way. * * allow_always and reject_always are never chosen: either would be remembered * by the agent past this grant, so the next call would skip the check. * An allow rule with no allow_once option therefore denies. */ import { traceOf } from '../trace/events.js'; +import { DECIDED_BY_ROLES, isUserId } from '../record/format.js'; import { matchToolCall, validateGrant } from './grant.js'; export const DEFAULT_APPROVE_TIMEOUT_MS = 5 * 60 * 1000; @@ -28,14 +32,34 @@ const denyAnswer = (options) => { return option ? { outcome: 'selected', optionId: option.optionId, kind: option.kind } : { outcome: 'cancelled' }; }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isText = (value) => typeof value === 'string' && value.length > 0; + +/** + * What an approver's answer says. Only `true`, or an object with + * `approved === true`, approves; attribution is taken from an object either + * way, and a malformed field is dropped rather than trusted. + */ +const readAnswer = (answer) => { + if (answer === true) return { verdict: 'approved' }; + if (!isPlainObject(answer)) return { verdict: 'rejected' }; + return { + verdict: answer.approved === true ? 'approved' : 'rejected', + ...(isUserId(answer.by) ? { by: answer.by } : {}), + ...(DECIDED_BY_ROLES.includes(answer.role) ? { role: answer.role } : {}), + ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), + }; +}; + /** - * Wait for the host's approval, bounded. Resolves 'approved' | 'rejected' | - * 'timed_out' | 'failed' | 'aborted'; never rejects, so a broken approver - * becomes a denial rather than a hung or crashed turn. + * Wait for the host's approval, bounded. Resolves { verdict, by?, role?, + * decisionCardId? }, verdict 'approved' | 'rejected' | 'timed_out' | 'failed' + * | 'aborted'; never rejects, so a broken approver becomes a denial rather + * than a hung or crashed turn. */ const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((done) => { - if (typeof approve !== 'function') { done('failed'); return; } - if (signal?.aborted) { done('aborted'); return; } + if (typeof approve !== 'function') { done({ verdict: 'failed' }); return; } + if (signal?.aborted) { done({ verdict: 'aborted' }); return; } let finished = false; const finish = (verdict) => { if (finished) return; @@ -44,23 +68,42 @@ const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((do signal?.removeEventListener?.('abort', onAbort); done(verdict); }; - const onAbort = () => finish('aborted'); - const timer = setTimeout(() => finish('timed_out'), timeoutMs); + const onAbort = () => finish({ verdict: 'aborted' }); + const timer = setTimeout(() => finish({ verdict: 'timed_out' }), timeoutMs); signal?.addEventListener?.('abort', onAbort, { once: true }); Promise.resolve() .then(() => approve(input)) - .then((answer) => finish(answer === true ? 'approved' : 'rejected'), () => finish('failed')); + .then((answer) => finish(readAnswer(answer)), () => finish({ verdict: 'failed' })); }); /** - * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal? }) + * Who decided an ask, for the record. A person who answered is named by the + * approver; no answer (timeout, failure, cancelled run) means the policy's + * default decided. A bare true/false names nobody: decidedBy null. + * Rule-decided calls carry no decidedBy here; the record writer fills in + * "policy" for any decision whose rule list is not `ask`. + */ +const decider = (approval) => { + if (approval.verdict === 'approved' || approval.verdict === 'rejected') { + return { + decidedBy: approval.by ?? null, + decidedByRole: approval.role ?? null, + ...(approval.decisionCardId ? { decisionCardId: approval.decisionCardId } : {}), + }; + } + return { decidedBy: 'policy', decidedByRole: 'policy' }; +}; + +/** + * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal?, grantId? }) * → (request) => Promise<{outcome:'selected', optionId} | {outcome:'cancelled'}> * * `signal` is the run's AbortSignal: a cancelled run answers a pending ask * with cancelled, which is what ACP asks of a client after session/cancel. + * `grantId` is recorded on each decision, to tie it to its grant.issued. */ export const createPermissionPolicy = (grant, { - approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, + approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, grantId, } = {}) => { const valid = validateGrant(grant); const hook = traceOf({ trace }); @@ -72,8 +115,8 @@ export const createPermissionPolicy = (grant, { let answer; if (decision === 'ask') { approval = await awaitApproval(approve, { toolCall, grantRule: rule }, { timeoutMs: approveTimeoutMs, signal }); - if (approval === 'aborted') answer = { outcome: 'cancelled' }; - else answer = approval === 'approved' ? allowAnswer(options) : null; + if (approval.verdict === 'aborted') answer = { outcome: 'cancelled' }; + else answer = approval.verdict === 'approved' ? allowAnswer(options) : null; } else if (decision === 'allow') { answer = allowAnswer(options); } @@ -83,7 +126,8 @@ export const createPermissionPolicy = (grant, { ...(answer.optionId ? { optionId: answer.optionId, kind: answer.kind } : {}), toolCallId: typeof toolCall.toolCallId === 'string' ? toolCall.toolCallId : null, rule, - ...(approval ? { approval } : {}), + ...(approval ? { approval: approval.verdict, ...decider(approval) } : {}), + ...(grantId ? { grantId } : {}), }); return answer.outcome === 'selected' ? { outcome: 'selected', optionId: answer.optionId } : { outcome: 'cancelled' }; }; diff --git a/packages/seats/test-support/record-sample.js b/packages/seats/test-support/record-sample.js new file mode 100644 index 0000000000..cc962c82f1 --- /dev/null +++ b/packages/seats/test-support/record-sample.js @@ -0,0 +1,142 @@ +/** + * Builds docs/examples: one task, two people, two agents on different CLIs. + * + * Ana's opencode agent fixes TASK-101 under a grant that attenuates Ben's + * Sentry connection grant. Its first attempt hangs and is retried; a + * `git push` is denied by the grant; a Sentry write asks, and Ben approves + * it as the credential's owner through a decision card. Ben's goose agent + * then reviews the PR, and the board marks the task done, which ends the + * review grant bound to it. + * + * Everything runs through the real store and valet, with a fixed clock and + * fake credential values, so the files are reproducible byte for byte. + */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { toOtlpJson } from '../src/record/otel.js'; + +export const SAMPLE_SECRETS = Object.freeze(['sk-or-sample-0000aaaa1111bbbb', 'sntry-proxy-sample-2222cccc3333', 'sk-sample-openai-4444dddd5555']); +export const SAMPLE_RUN_IDS = Object.freeze(['run-ana-0001', 'run-ben-0001']); +const POD = 'pod-checkout'; +const TASK = 'TASK-101'; +const OPTIONS = [ + { optionId: 'allow', name: 'Allow', kind: 'allow_once' }, + { optionId: 'always', name: 'Always allow', kind: 'allow_always' }, + { optionId: 'reject', name: 'Reject', kind: 'reject_once' }, +]; +const options = OPTIONS.map(({ optionId, kind }) => ({ optionId, kind })); + +/** A tool call the way an ACP adapter would record it, decided by the run's policy. */ +const toolCall = async (run, decide, call, finalStatus) => { + run.emit('tool.call', { ...call, status: 'pending' }); + run.emit('permission.requested', { toolCallId: call.toolCallId, options }); + const answer = await decide({ sessionId: 'ses', toolCall: { toolCallId: call.toolCallId, title: call.title, kind: call.kind }, options: OPTIONS }); + const allowed = answer.outcome === 'selected' && answer.optionId === 'allow'; + run.emit('tool.update', { toolCallId: call.toolCallId, status: allowed ? finalStatus : 'failed' }); +}; + +export const buildSample = async (root) => { + let tick = Date.UTC(2026, 9, 25, 16, 0, 0); + const store = createTraceStore({ root: join(root, 'trace'), now: () => new Date((tick += 1000)) }); + const [secretOr, secretSentry, secretOpenai] = SAMPLE_SECRETS; + + // Ana's agent: opencode, retried once, one denied call, one approved by Ben. + const ana = store.startRun({ + runId: SAMPLE_RUN_IDS[0], adapter: 'opencode', taskId: TASK, podId: POD, + actor: { type: 'agent', id: 'u-agent-ana-opencode', owner: 'u-ana', name: "Ana's opencode" }, + }); + ana.emit('task.claimed', { leaseUntil: '2026-10-25T18:00:00Z' }); + const anaGrant = { + credentials: [{ name: 'OPENROUTER_API_KEY', from: 'env' }, { name: 'SENTRY_PROXY_TOKEN', from: 'resolver' }], + tools: { allow: ['read', 'search', 'other:commonly_*'], ask: ['fetch'], deny: ['execute:git push*'] }, + }; + const anaEnv = await prepareRunEnv({ + grant: anaGrant, + baseEnv: { OPENROUTER_API_KEY: secretOr }, + resolveSecret: async () => secretSentry, + root, + ctx: { trace: ana }, + grantMeta: { + grantId: 'grant-ana-sentry-1', + parentGrantId: 'grant-ben-sentry-root', + scope: { + connectionId: 'conn-ben-sentry', tools: ['sentry.get_issue', 'sentry.resolve_issue'], writeMode: 'write-with-confirm', + budget: { calls: 20, windowMs: 3600000 }, audience: ['u-ana', 'u-ben'], + }, + target: { kind: 'seat', id: 'u-agent-ana-opencode' }, + issuedBy: 'u-ana', + expiresAt: '2026-10-25T20:00:00.000Z', + taskId: TASK, + podId: POD, + }, + }); + await anaEnv.cleanup(); + const anaDecide = createPermissionPolicy(anaGrant, { + trace: ana, + grantId: 'grant-ana-sentry-1', + approve: async () => ({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-7f3' }), + }); + + ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, attempt: 1 }); + ana.emit('turn.prompt_sent', { chars: 412 }); + ana.emit('tool.call', { toolCallId: 'tc-1', title: 'Read src/checkout/total.js', kind: 'read', status: 'pending', surface: 'native' }); + ana.emit('tool.update', { toolCallId: 'tc-1', status: 'completed' }); + ana.emit('liveness.stalled', { idleMs: 90000, windowMs: 90000 }); + ana.emit('liveness.recovery', { action: 'cancel', attempt: 1 }); + ana.emit('attempt.settled', { attempt: 1, outcome: 'stuck', failure: { kind: 'stuck', detail: 'no progress signal for 90000ms (window 90000ms)' } }); + ana.emit('liveness.recovery', { action: 'retry', attempt: 2 }); + + ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, sessionId: 'ses-ana-2', attempt: 2 }); + await toolCall(ana, anaDecide, { toolCallId: 'tc-2', title: 'git push origin fix/total-rounding', kind: 'execute', surface: 'native' }, 'completed'); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-3', title: 'sentry.resolve_issue CHECKOUT-88', kind: 'fetch', name: 'sentry.resolve_issue', surface: 'connector', + }, 'completed'); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-4', title: 'commonly_post_message', kind: 'other', name: 'commonly_post_message', surface: 'commonly', + }, 'completed'); + ana.emit('agent.message_chunk', { text: 'Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open.' }); + ana.emit('usage', { inputTokens: 18234, outputTokens: 2210 }); + ana.emit('attempt.settled', { attempt: 2, outcome: 'ok' }); + ana.emit('result.reported', { status: 'done', refs: [{ type: 'pr', id: '#212' }] }); + ana.emit('run.settled', { outcome: 'ok', durationMs: 412000, attempts: 2 }); + + // Ben's agent: goose, read-only review; the board then marks the task done. + const ben = store.startRun({ + runId: SAMPLE_RUN_IDS[1], adapter: 'goose', taskId: TASK, podId: POD, + actor: { type: 'agent', id: 'u-agent-ben-goose', owner: 'u-ben', name: "Ben's goose" }, + }); + const benGrant = { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }], tools: { allow: ['read', 'search'] } }; + const benEnv = await prepareRunEnv({ + grant: benGrant, + baseEnv: { OPENAI_API_KEY: secretOpenai }, + root, + ctx: { trace: ben }, + grantMeta: { + grantId: 'grant-ben-review-1', + scope: { connectionId: 'conn-ben-github', tools: ['github.get_pull_request'], writeMode: 'read', audience: ['u-ben'] }, + target: { kind: 'seat', id: 'u-agent-ben-goose' }, + issuedBy: 'u-ben', + expiresAt: '2026-10-25T20:00:00.000Z', + taskId: TASK, + podId: POD, + }, + }); + await benEnv.cleanup(); + const benDecide = createPermissionPolicy(benGrant, { trace: ben, grantId: 'grant-ben-review-1' }); + ben.emit('run.started', { adapter: 'goose', cli: { name: 'goose', version: '1.54.0' }, attempt: 1 }); + await toolCall(ben, benDecide, { + toolCallId: 'tc-1', title: 'github.get_pull_request #212', kind: 'read', name: 'github.get_pull_request', surface: 'connector', + }, 'completed'); + ben.emit('agent.message_chunk', { text: 'Reviewed #212: the rounding fix is right and the half-cent case is tested.' }); + ben.emit('attempt.settled', { attempt: 1, outcome: 'ok' }); + ben.emit('task.done', { doneBy: 'u-ben' }); + ben.emit('run.settled', { outcome: 'ok', durationMs: 95000, attempts: 1 }); + + const files = Object.fromEntries(SAMPLE_RUN_IDS.map((runId) => [`${runId}.jsonl`, readFileSync(join(root, 'trace', 'runs', `${runId}.jsonl`), 'utf8')])); + const entries = SAMPLE_RUN_IDS.flatMap((runId) => store.readRun(runId)); + files[`${TASK}.otlp.json`] = `${JSON.stringify(toOtlpJson(entries, { serviceName: 'seats-sample' }), null, 2)}\n`; + return { store, files, entries }; +}; diff --git a/packages/seats/test/record-example.test.js b/packages/seats/test/record-example.test.js new file mode 100644 index 0000000000..6bb16a991f --- /dev/null +++ b/packages/seats/test/record-example.test.js @@ -0,0 +1,71 @@ +/** + * docs/examples is generated from test-support/record-sample.js. This test + * fails when the files drift from what the code writes; regenerate them with + * SEATS_WRITE_EXAMPLES=1 node --test test/record-example.test.js + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildSample, SAMPLE_RUN_IDS, SAMPLE_SECRETS } from '../test-support/record-sample.js'; +import { verifyRecord } from '../src/record/format.js'; + +const EXAMPLES = join(dirname(fileURLToPath(import.meta.url)), '..', 'docs', 'examples'); + +const built = async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-example-')); + try { + const { files, store } = await buildSample(root); + return { files, verified: SAMPLE_RUN_IDS.map((runId) => store.verifyRun(runId)) }; + } finally { rmSync(root, { recursive: true, force: true }); } +}; + +test('docs/examples matches what the code writes', async () => { + const { files, verified } = await built(); + for (const result of verified) assert.deepEqual(result.errors, []); + if (process.env.SEATS_WRITE_EXAMPLES === '1') { + mkdirSync(EXAMPLES, { recursive: true }); + for (const [name, body] of Object.entries(files)) writeFileSync(join(EXAMPLES, name), body); + } + for (const [name, body] of Object.entries(files)) { + const path = join(EXAMPLES, name); + assert.ok(existsSync(path), `${name} is missing; run with SEATS_WRITE_EXAMPLES=1`); + assert.equal(readFileSync(path, 'utf8'), body, `${name} drifted; run with SEATS_WRITE_EXAMPLES=1`); + } +}); + +test('the committed examples verify on their own and hold no secret value', () => { + for (const runId of SAMPLE_RUN_IDS) { + const text = readFileSync(join(EXAMPLES, `${runId}.jsonl`), 'utf8'); + const result = verifyRecord(text); + assert.deepEqual(result.errors, []); + assert.equal(result.settled, true); + } + const all = SAMPLE_RUN_IDS.map((id) => `${id}.jsonl`).concat('TASK-101.otlp.json') + .map((name) => readFileSync(join(EXAMPLES, name), 'utf8')).join('\n'); + for (const secret of SAMPLE_SECRETS) assert.ok(!all.includes(secret)); +}); + +test('the headline case: Ana\'s agent asks, Ben approves as the credential owner', () => { + const entries = readFileSync(join(EXAMPLES, 'run-ana-0001.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + const asked = entries.find((e) => e.kind === 'permission.decided' && e.data.approval === 'approved'); + const agent = entries.find((e) => e.kind === 'attempt.started').actor; + assert.equal(agent.owner, 'u-ana'); + assert.equal(asked.data.decidedBy, 'u-ben'); + assert.notEqual(asked.data.decidedBy, agent.owner, 'the approver is not the agent\'s owner'); + assert.equal(asked.data.decidedByRole, 'credential-owner'); + assert.equal(asked.data.decisionCardId, 'card-7f3'); + assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }); + const issued = entries.find((e) => e.kind === 'grant.issued'); + assert.equal(asked.data.grantId, issued.data.grantId); + assert.equal(issued.data.scope.connectionId, 'conn-ben-sentry'); + assert.equal(issued.data.parentGrantId, 'grant-ben-sentry-root'); + const denied = entries.find((e) => e.kind === 'permission.decided' && e.data.rule.list === 'deny'); + assert.deepEqual([denied.data.decidedBy, denied.data.decidedByRole], ['policy', 'policy']); + const ben = readFileSync(join(EXAMPLES, 'run-ben-0001.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + const revoked = ben.find((e) => e.kind === 'grant.revoked'); + assert.deepEqual(revoked.data, { grantIds: ['grant-ben-review-1'], reason: 'task.done' }); + assert.equal(ben[ben.indexOf(revoked) - 1].kind, 'task.done'); +}); diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js new file mode 100644 index 0000000000..7cd6b10777 --- /dev/null +++ b/packages/seats/test/record-format.test.js @@ -0,0 +1,254 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { + appendEntry, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, +} from '../src/record/format.js'; + +const RUN = 'run-fmt-0001'; +const ENV = { runId: RUN, taskId: 'TASK-7', podId: 'pod-a' }; +const AGENT = { type: 'agent', id: 'u-agent-1', owner: 'u-ana', name: "Ana's agent" }; +let clock = Date.UTC(2026, 9, 25, 12, 0, 0); +const ts = () => new Date((clock += 1000)).toISOString(); + +/** A small, settled record built through the writer: [lines], each without '\n'. */ +const build = (count = 6) => { + const lines = []; + const add = (kind, data, extra = {}) => { + lines.push(appendEntry(lines.at(-1) ?? null, { ...ENV, ts: ts(), kind, data, ...extra }, { runActor: AGENT })); + }; + add('task.claimed', {}); + add('attempt.started', { attempt: 1, adapter: 'fake' }); + for (let i = 0; i < count - 3; i += 1) add('tool.call', { toolCallId: `tc-${i}`, title: `step ${i}`, surface: 'native' }); + add('run.settled', { outcome: 'ok', durationMs: 10, attempts: 1 }); + return lines; +}; +const text = (lines) => lines.map((line) => `${line}\n`).join(''); +const reasons = (result) => result.errors.map(({ seq, reason }) => `${seq}:${reason}`); + +test('canonical form: sorted keys, no whitespace, absent fields omitted', () => { + assert.equal(canonicalLine({ b: 1, a: { d: [1, 'x'], c: null }, e: undefined }), '{"a":{"c":null,"d":[1,"x"]},"b":1}'); + // UTF-16 code-unit order, as RFC 8785 asks: 'B' (0x42) sorts before 'a' (0x61). + assert.equal(canonicalLine({ a: 1, B: 2 }), '{"B":2,"a":1}'); + assert.throws(() => canonicalLine({ x: Number.NaN }), /finite/); +}); + +test('round trip: what the writer wrote verifies, reads back, and chains by the exact bytes', () => { + const lines = build(); + const result = verifyRecord(text(lines)); + assert.deepEqual(result.errors, []); + assert.equal(result.ok, true); + assert.equal(result.entries, lines.length); + assert.equal(result.settled, true); + assert.equal(result.head, createHash('sha256').update(lines.at(-1)).digest('hex')); + const entries = readRecord(text(lines)); + assert.equal(entries.length, lines.length); + entries.forEach((entry, i) => { + assert.equal(entry.v, RECORD_VERSION); + assert.equal(entry.schema, RECORD_SCHEMA); + assert.equal(entry.seq, i + 1); + assert.equal(entry.prev, i === 0 ? null : hashLine(lines[i - 1])); + assert.equal(canonicalLine(entry), lines[i], 'each line is the canonical form of its entry'); + assert.equal(entry.taskId, 'TASK-7'); + assert.equal(entry.podId, 'pod-a'); + }); + assert.deepEqual(entries[0].actor, { type: 'agent', id: 'u-agent-1', owner: 'u-ana', name: "Ana's agent" }); + assert.deepEqual(entries.at(-1).actor, { type: 'system', id: 'seats' }, 'run.settled defaults to the system actor'); +}); + +test('tamper: an edited byte is reported at the next seq as prev_mismatch', () => { + const lines = build(); + // A byte inside seq 3's data: still JSON, still canonical, so only the chain sees it. + lines[2] = lines[2].replace('step 0', 'step 9'); + const result = verifyRecord(text(lines)); + assert.equal(result.ok, false); + assert.deepEqual(reasons(result), ['4:prev_mismatch']); +}); + +test('tamper: an edited byte that breaks canonical form is reported on its own line too', () => { + const lines = build(); + lines[2] = lines[2].replace('"kind":"tool.call"', '"kind": "tool.call"'); + assert.deepEqual(reasons(verifyRecord(text(lines))), ['3:not_canonical', '4:prev_mismatch']); +}); + +test('tamper: a deleted line is a seq gap and a prev mismatch at the line after it', () => { + const lines = build(); + lines.splice(2, 1); // seq 3 gone + const result = verifyRecord(text(lines)); + assert.deepEqual(reasons(result), ['4:seq_mismatch', '4:prev_mismatch']); +}); + +test('tamper: deleting the first line is caught at seq 2', () => { + const lines = build(); + lines.shift(); + assert.deepEqual(reasons(verifyRecord(text(lines))), ['2:seq_mismatch', '2:prev_mismatch']); +}); + +test('tamper: swapped lines are reported where the order breaks', () => { + const lines = build(); + [lines[2], lines[3]] = [lines[3], lines[2]]; // seq 4 now sits before seq 3 + const result = verifyRecord(text(lines)); + assert.equal(result.ok, false); + assert.deepEqual(reasons(result), ['4:seq_mismatch', '4:prev_mismatch', '3:seq_mismatch', '3:prev_mismatch', '5:seq_mismatch', '5:prev_mismatch']); +}); + +test('tamper: a torn final line is reported at the seq it would have had', () => { + const lines = build(); + const torn = `${text(lines.slice(0, -1))}${lines.at(-1).slice(0, 25)}`; + const result = verifyRecord(torn); + assert.deepEqual(reasons(result), [`${lines.length}:torn`]); + assert.equal(result.entries, lines.length - 1); + assert.equal(result.settled, false); + assert.equal(readRecord(torn).length, lines.length - 1, 'readers skip the torn tail'); +}); + +test('a record cut short at a line boundary is an unsettled prefix, not tampering', () => { + const result = verifyRecord(text(build().slice(0, 3))); + assert.equal(result.ok, true); + assert.equal(result.settled, false); +}); + +test('a line after run.settled, and a line from another run, are reported', () => { + const lines = build(); + const extra = JSON.parse(lines.at(-1)); + const after = canonicalLine({ ...extra, seq: extra.seq + 1, kind: 'usage', data: {}, prev: hashLine(lines.at(-1)) }); + assert.deepEqual(reasons(verifyRecord(text([...lines, after]))), [`${extra.seq + 1}:after_settled`]); + assert.throws(() => appendEntry(lines.at(-1), { ...ENV, ts: ts(), kind: 'usage', data: {} }), /after run.settled/); + const mid = JSON.parse(lines[1]); + lines[1] = canonicalLine({ ...mid, runId: 'run-other-01' }); + assert.ok(reasons(verifyRecord(text(lines))).includes('2:run_mismatch')); +}); + +test('a newer v is refused, and unknown kinds still chain', () => { + const first = JSON.parse(build()[0]); + const v2 = canonicalLine({ ...first, v: 2 }); + assert.deepEqual(reasons(verifyRecord(`${v2}\n`)), ['1:unsupported_version']); + const unknown = canonicalLine({ ...first, kind: 'future.thing' }); + const next = canonicalLine({ ...JSON.parse(build()[1]), prev: hashLine(unknown) }); + assert.deepEqual(verifyRecord(`${unknown}\n${next}\n`).errors, []); +}); + +test('podId is required whenever taskId is set: writer and verifier', () => { + assert.throws(() => appendEntry(null, { runId: RUN, taskId: 'TASK-7', ts: ts(), kind: 'task.claimed', data: {} }), /taskId needs a podId/); + // A podId alone is fine; so is neither. + appendEntry(null, { runId: RUN, podId: 'pod-a', ts: ts(), kind: 'task.claimed', data: {} }); + appendEntry(null, { runId: RUN, ts: ts(), kind: 'attempt.started', data: { attempt: 1 } }); + const line = canonicalLine({ + v: 1, schema: RECORD_SCHEMA, runId: RUN, taskId: 'TASK-7', seq: 1, ts: ts(), kind: 'task.claimed', actor: null, data: {}, prev: null, + }); + assert.deepEqual(verifyRecord(`${line}\n`).errors, [{ seq: 1, reason: 'bad_envelope', detail: 'podId' }]); +}); + +test('taskId and podId are fixed for the run', () => { + const first = appendEntry(null, { ...ENV, ts: ts(), kind: 'task.claimed', data: {} }); + assert.throws(() => appendEntry(first, { ...ENV, podId: 'pod-b', ts: ts(), kind: 'usage', data: {} }), /fixed for the whole run/); +}); + +test('actor.id is an id, never a display name; owner is a user id on agents only', () => { + const at = (actor) => appendEntry(null, { runId: RUN, ts: ts(), kind: 'usage', data: {}, actor }); + assert.throws(() => at({ type: 'agent', id: "Ana's agent" }), /actor.id/); + assert.throws(() => at({ type: 'human', id: 'Ben Smith' }), /actor.id/); + assert.throws(() => at({ type: 'human', id: 'u-ben', owner: 'u-ana' }), /owner/); + assert.throws(() => at({ type: 'robot', id: 'u-x' }), /actor.type/); + const entry = JSON.parse(at({ type: 'human', id: 'u-ben', name: 'Ben Smith' })); + assert.deepEqual(entry.actor, { id: 'u-ben', name: 'Ben Smith', type: 'human' }); + assert.equal(JSON.parse(at(null)).actor, null, 'null records "unknown" and is kept'); +}); + +const grantData = (overrides = {}) => ({ + grantId: 'g-1', + scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], writeMode: 'read', budget: { calls: 5, windowMs: 60000 }, audience: ['u-ana'] }, + issuedBy: 'u-ana', + ...overrides, +}); +const issue = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'grant.issued', data }); + +test('grant.issued: scope matches a room grant, with exact tool names and a connection id', () => { + const entry = JSON.parse(issue(grantData({ + parentGrantId: 'g-root', target: { kind: 'seat', id: 'u-agent-1' }, expiresAt: '2026-10-26T00:00:00Z', + binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.done' }, + }))); + assert.deepEqual(entry.data.scope, grantData().scope); + assert.equal(entry.data.parentGrantId, 'g-root'); + assert.deepEqual(entry.actor, { type: 'human', id: 'u-ana' }, 'the issuer is the default actor'); + // Patterns, a credential name in place of a connection, and unknown fields are refused. + for (const tools of [['sentry.*'], ['execute:git push*'], ['read?'], ['two words']]) { + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, tools } })), /exact tool names/, `tools ${tools}`); + } + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, credentials: ['SENTRY_TOKEN'] } })), /unknown field grant.issued.scope.credentials/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, writeMode: 'admin' } })), /writeMode/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, budget: { calls: -1 } } })), /budget.calls/); + assert.throws(() => issue(grantData({ scope: { ...grantData().scope, audience: ['Ana Smith'] } })), /audience/); + assert.throws(() => issue(grantData({ secret: 'x' })), /unknown field grant.issued.secret/); + assert.equal(JSON.parse(issue(grantData({ issuedBy: 'policy' }))).actor.id, 'grant:g-1'); +}); + +test('grant.issued binding: names the run\'s own task and pod, and ends on task.done or expiresAt', () => { + issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.done' } })); + issue(grantData({ expiresAt: '2026-10-26T00:00:00Z', binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'expiresAt' } })); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'task.closed' } })), /endsOn/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', podId: 'pod-a', endsOn: 'expiresAt' } })), /needs an expiresAt/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-7', endsOn: 'task.done' } })), /taskId and podId/); + assert.throws(() => issue(grantData({ binding: { taskId: 'TASK-8', podId: 'pod-a', endsOn: 'task.done' } })), /run's own/); +}); + +test('grant.revoked: a non-empty list of grant ids and a reason from the enum', () => { + const revoke = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'grant.revoked', data }); + for (const reason of ['task.done', 'expired', 'cascade', 'manual']) revoke({ grantIds: ['g-1', 'g-2'], reason }); + assert.throws(() => revoke({ grantIds: [], reason: 'manual' }), /non-empty/); + assert.throws(() => revoke({ grantId: 'g-1', reason: 'manual' }), /grantIds/); + assert.throws(() => revoke({ grantIds: ['g-1', 'g-1'], reason: 'manual' }), /repeats/); + for (const reason of ['task.closed', 'run.settled', 'revoked_by:u-ana', undefined]) { + assert.throws(() => revoke({ grantIds: ['g-1'], reason }), /reason/, `reason ${reason}`); + } +}); + +test('task.done: written by the board, and only in a run with a task', () => { + const entry = JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'task.done', data: { doneBy: 'u-ben' } })); + assert.deepEqual(entry.actor, { type: 'system', id: 'board' }); + assert.throws(() => appendEntry(null, { runId: RUN, ts: ts(), kind: 'task.done', data: {} }), /needs a run with a taskId/); + assert.equal(KINDS['task.done'].actor, 'board'); +}); + +test('permission.decided: decidedBy is separate from the agent owner, and credential-owner is a role', () => { + const decide = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data }, { runActor: AGENT })); + const asked = decide({ + outcome: 'selected', optionId: 'allow', kind: 'allow_once', toolCallId: 'tc-1', rule: { list: 'ask', pattern: 'fetch' }, + approval: 'approved', decidedBy: 'u-ben', decidedByRole: 'credential-owner', decisionCardId: 'card-1', + }); + assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }, 'the decider is the actor, not the agent or its owner'); + assert.equal(asked.data.decidedByRole, 'credential-owner'); + for (const role of ['decision-owner', 'agent-owner', 'policy']) decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: role }); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'owner' }), /decidedByRole/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'Ben Smith' }), /decidedBy/); + const byRule = decide({ outcome: 'selected', optionId: 'reject', kind: 'reject_once', toolCallId: 't', rule: { list: 'deny', pattern: 'x' }, grantId: 'g-1' }); + assert.equal(byRule.data.decidedBy, 'policy'); + assert.equal(byRule.data.decidedByRole, 'policy'); + assert.deepEqual(byRule.actor, { type: 'policy', id: 'grant:g-1' }); + const unnamed = decide({ outcome: 'selected', optionId: 'allow', toolCallId: 't', rule: { list: 'ask', pattern: 'x' }, approval: 'approved' }); + assert.equal(unnamed.data.decidedBy, null, 'an ask with no named approver stays null'); + assert.equal(unnamed.actor, null); +}); + +test('tool.call surface is one of connector, commonly, native', () => { + const call = (surface) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', surface } }); + for (const surface of ['connector', 'commonly', 'native']) assert.equal(JSON.parse(call(surface)).data.surface, surface); + assert.throws(() => call('mcp'), /surface/); + assert.equal(toolSurfaceOf('commonly_post_message'), 'commonly'); + assert.equal(toolSurfaceOf('mcp__commonly__commonly_read'), 'commonly'); + assert.equal(toolSurfaceOf('mcp__sentry__resolve_issue'), 'connector'); + assert.equal(toolSurfaceOf('Bash'), 'native'); +}); + +test('credential entries hold names only', () => { + const granted = (names) => appendEntry(null, { ...ENV, ts: ts(), kind: 'credential.granted', data: { names } }); + granted(['OPENAI_API_KEY']); + assert.throws(() => granted(['sk-live-0123456789']), /NAMES/); + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'credential.denied', data: { name: 'sk-abc', reason: 'unresolved' } }), /NAMES/); +}); + +test('the writer refuses unknown kinds, a bad seq, and a non-UTC ts', () => { + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'made.up', data: {} }), /unknown record kind/); + assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'usage', data: {}, seq: 2 }), /seq 2/); + assert.throws(() => appendEntry(null, { ...ENV, ts: '2026-10-25T12:00:00+02:00', kind: 'usage', data: {} }), /RFC 3339/); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js new file mode 100644 index 0000000000..0c52f346b4 --- /dev/null +++ b/packages/seats/test/record-otel.test.js @@ -0,0 +1,131 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { toOtlpJson } from '../src/record/otel.js'; +import { buildSample } from '../test-support/record-sample.js'; + +const sample = async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-otel-')); + try { return await buildSample(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const spansOf = (otlp) => otlp.resourceSpans[0].scopeSpans[0].spans; +const attrOf = (holder, key) => holder.attributes.find((a) => a.key === key)?.value; +const ANY_VALUE_KEYS = new Set(['stringValue', 'boolValue', 'intValue', 'doubleValue', 'arrayValue', 'kvlistValue']); + +const checkAnyValue = (value, where) => { + const keys = Object.keys(value); + assert.equal(keys.length, 1, `${where}: one AnyValue field`); + assert.ok(ANY_VALUE_KEYS.has(keys[0]), `${where}: ${keys[0]}`); + if (keys[0] === 'intValue') assert.match(value.intValue, /^-?\d+$/, `${where}: intValue is a decimal string in OTLP/JSON`); + if (keys[0] === 'arrayValue') value.arrayValue.values.forEach((v, i) => checkAnyValue(v, `${where}[${i}]`)); + if (keys[0] === 'kvlistValue') value.kvlistValue.values.forEach((kv) => checkAnyValue(kv.value, `${where}.${kv.key}`)); +}; + +test('OTLP/JSON structure: ids, times, kinds, typed attributes', async () => { + const { entries } = await sample(); + const otlp = toOtlpJson(entries, { serviceName: 'seats-test' }); + assert.equal(otlp.resourceSpans.length, 1); + assert.deepEqual(otlp.resourceSpans[0].resource.attributes, [{ key: 'service.name', value: { stringValue: 'seats-test' } }]); + assert.deepEqual(otlp.resourceSpans[0].scopeSpans[0].scope, { name: 'seats.record', version: '1' }); + const spans = spansOf(otlp); + assert.ok(spans.length > 0); + const ids = new Set(); + for (const span of spans) { + assert.match(span.traceId, /^[0-9a-f]{32}$/); + assert.match(span.spanId, /^[0-9a-f]{16}$/); + assert.ok(!/^0+$/.test(span.traceId) && !/^0+$/.test(span.spanId)); + assert.ok(!ids.has(span.spanId), `span ids are unique: ${span.name}`); + ids.add(span.spanId); + if (span.parentSpanId !== undefined) assert.match(span.parentSpanId, /^[0-9a-f]{16}$/); + assert.equal(span.kind, 1, 'INTERNAL'); + assert.match(span.startTimeUnixNano, /^\d+$/); + assert.match(span.endTimeUnixNano, /^\d+$/, `${span.name} has an end`); + assert.ok(BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano)); + assert.ok([0, 2].includes(span.status.code)); + for (const a of span.attributes) checkAnyValue(a.value, `${span.name}.${a.key}`); + for (const event of span.events) { + assert.match(event.timeUnixNano, /^\d+$/); + for (const a of event.attributes) checkAnyValue(a.value, `${span.name}/${event.name}.${a.key}`); + } + } + // Content stays out: no message text anywhere in the export. + assert.ok(!JSON.stringify(otlp).includes('half-cent rounding')); +}); + +test('one task is one trace; ids follow the documented derivation', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const expectedTrace = createHash('sha256').update('seats.record/trace/task/pod-checkout/TASK-101').digest('hex').slice(0, 32); + assert.deepEqual([...new Set(spans.map((s) => s.traceId))], [expectedTrace]); + const runSpan = spans.find((s) => s.name === 'seats.run' && attrOf(s, 'seats.run.id').stringValue === 'run-ana-0001'); + assert.equal(runSpan.spanId, createHash('sha256').update('seats.record/span/run-ana-0001/0').digest('hex').slice(0, 16)); + const first = entries.find((e) => e.runId === 'run-ana-0001' && e.kind === 'attempt.started'); + const attempt = spans.find((s) => s.spanId === createHash('sha256').update(`seats.record/span/run-ana-0001/${first.seq}`).digest('hex').slice(0, 16)); + assert.ok(attempt.name.startsWith('invoke_agent')); +}); + +test('parent/child links: run → attempt → tool, and decisions sit on their tool span', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const byId = new Map(spans.map((s) => [s.spanId, s])); + const runs = spans.filter((s) => s.name === 'seats.run'); + assert.equal(runs.length, 2); + for (const run of runs) assert.equal(run.parentSpanId, undefined, 'a run span is a root'); + const attempts = spans.filter((s) => s.name.startsWith('invoke_agent')); + assert.equal(attempts.length, 3, 'two attempts for Ana, one for Ben'); + for (const attempt of attempts) { + assert.equal(byId.get(attempt.parentSpanId).name, 'seats.run'); + assert.equal(attrOf(attempt, 'gen_ai.operation.name').stringValue, 'invoke_agent'); + } + const tools = spans.filter((s) => s.name.startsWith('execute_tool')); + for (const tool of tools) { + assert.ok(byId.get(tool.parentSpanId).name.startsWith('invoke_agent'), `${tool.name} is a child of an attempt`); + assert.equal(attrOf(tool, 'gen_ai.operation.name').stringValue, 'execute_tool'); + } + const resolve = tools.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + const parent = byId.get(resolve.parentSpanId); + assert.equal(attrOf(parent, 'seats.attempt.number').intValue, '2', 'the approved call ran in the retry'); + assert.equal(attrOf(resolve, 'gen_ai.tool.call.id').stringValue, 'tc-3'); + assert.equal(attrOf(resolve, 'seats.tool.surface').stringValue, 'connector'); + const decided = resolve.events.find((e) => e.name === 'permission.decided'); + assert.equal(attrOf(decided, 'seats.decided_by').stringValue, 'u-ben'); + assert.equal(attrOf(decided, 'seats.decided_by_role').stringValue, 'credential-owner'); + assert.equal(attrOf(decided, 'seats.actor.id').stringValue, 'u-ben'); + assert.equal(attrOf(parent, 'seats.agent.owner').stringValue, 'u-ana'); + // The stuck first attempt is an error with its failure kind; the run is not. + const stuck = attempts.find((s) => attrOf(s, 'seats.attempt.number').intValue === '1' && attrOf(s, 'seats.agent.owner').stringValue === 'u-ana'); + assert.equal(stuck.status.code, 2); + assert.equal(attrOf(stuck, 'error.type').stringValue, 'stuck'); + const push = tools.find((s) => s.name.startsWith('execute_tool git push')); + assert.equal(push.status.code, 2); + assert.equal(runs.every((r) => r.status.code === 0), true); +}); + +test('the export is deterministic: same entries, same bytes, in any input order', async () => { + const a = await sample(); + const b = await sample(); + const one = JSON.stringify(toOtlpJson(a.entries)); + assert.equal(one, JSON.stringify(toOtlpJson(b.entries))); + const reversedPerRun = [...a.entries.filter((e) => e.runId === 'run-ana-0001').reverse(), ...a.entries.filter((e) => e.runId === 'run-ben-0001')]; + assert.equal(JSON.stringify(toOtlpJson(reversedPerRun)), one); +}); + +test('a run without a task is its own trace; unknown kinds and newer versions are skipped', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-solo-01', actor: null, prev: null }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', data: { attempt: 1 } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'future.kind', data: {} }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'run.settled', data: { outcome: 'failed', failure: { kind: 'crashed' } } }, + { ...base, v: 2, runId: 'run-v2-0001', seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', data: {} }, + ]; + const spans = spansOf(toOtlpJson(entries)); + assert.equal(spans.length, 2); + assert.equal(spans[0].traceId, createHash('sha256').update('seats.record/trace/run/run-solo-01').digest('hex').slice(0, 32)); + assert.ok(spans.every((s) => s.events.every((e) => e.name !== 'future.kind'))); + assert.equal(spans[0].status.code, 2); + assert.equal(attrOf(spans[0], 'error.type').stringValue, 'crashed'); + assert.equal(attrOf(spans[1], 'error.type').stringValue, 'crashed', 'an unsettled last attempt takes the run\'s failure'); +}); diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js new file mode 100644 index 0000000000..d1dd6c478c --- /dev/null +++ b/packages/seats/test/record-store.test.js @@ -0,0 +1,151 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { appendFileSync, chmodSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { verifyRecord } from '../src/record/format.js'; + +const AGENT = { type: 'agent', id: 'u-agent-ana', owner: 'u-ana', name: "Ana's agent" }; +const withStore = (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-')); + let tick = Date.UTC(2026, 9, 25, 12, 0, 0); + try { return fn(createTraceStore({ root, now: () => new Date((tick += 1000)) }), root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const fileOf = (root, runId) => join(root, 'runs', `${runId}.jsonl`); +const grantMeta = (grantId, extra = {}) => ({ + grantId, scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], audience: ['u-ana'] }, issuedBy: 'u-ana', ...extra, +}); + +test('readRun keeps the trace shape (type, data, runId, seq, ts) and adds the envelope', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0001', adapter: 'fake', taskId: 'TASK-9', podId: 'pod-a', actor: AGENT }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('agent.message_chunk', { text: 'hi' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + const events = store.readRun('run-st-0001'); + assert.deepEqual(events.map((e) => e.type), ['run.started', 'agent.message_chunk', 'run.settled']); + assert.deepEqual(events.map((e) => e.kind), ['attempt.started', 'agent.message_chunk', 'run.settled']); + assert.deepEqual(events.map((e) => e.seq), [1, 2, 3]); + for (const event of events) { + assert.equal(event.runId, 'run-st-0001'); + assert.equal(event.taskId, 'TASK-9'); + assert.equal(event.podId, 'pod-a'); + assert.equal(typeof event.ts, 'string'); + assert.equal(event.v, 1); + } + assert.deepEqual(events[0].actor, AGENT); + assert.deepEqual(events[0].data, { adapter: 'fake', attempt: 1 }); + const row = store.listRuns().find((r) => r.runId === 'run-st-0001'); + assert.equal(row.outcome, 'ok'); + assert.equal(row.taskId, 'TASK-9'); + assert.equal(row.entries, 3); + assert.equal(store.verifyRun('run-st-0001').ok, true); +})); + +test('startRun refuses a taskId without a podId, and a display name as actor id', () => withStore((store) => { + assert.throws(() => store.startRun({ runId: 'run-st-0002', taskId: 'TASK-9' }), /needs a podId/); + assert.throws(() => store.startRun({ runId: 'run-st-0003', actor: { type: 'agent', id: 'Ana agent' } }), /actor.id/); +})); + +test('emit: unknown types throw, and nothing is written after run.settled', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0004' }); + assert.throws(() => run.emit('made.up', {}), /unknown trace event type/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.equal(run.emit('usage', {}), null); + assert.equal(store.readRun('run-st-0004').length, 1); +})); + +test('files are 0600 and directories 0700', () => withStore((store, root) => { + store.startRun({ runId: 'run-st-0005' }).emit('usage', {}); + assert.equal(statSync(join(root, 'runs')).mode & 0o777, 0o700); + assert.equal(statSync(fileOf(root, 'run-st-0005')).mode & 0o777, 0o600); + assert.equal(statSync(join(root, 'index.jsonl')).mode & 0o777, 0o600); +})); + +test('verifyRun: a torn last line is reported', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0006' }); + run.emit('usage', {}); + run.emit('usage', {}); + appendFileSync(fileOf(root, 'run-st-0006'), '{"v":1,"schema":"seats.rec'); + const result = store.verifyRun('run-st-0006'); + assert.equal(result.ok, false); + assert.deepEqual(result.errors, [{ seq: 3, reason: 'torn' }]); + assert.equal(store.readRun('run-st-0006').length, 2, 'readRun still returns the whole lines'); +})); + +test('verifyRun: the index head catches an edited or dropped last line', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0007' }); + run.emit('usage', { inputTokens: 1 }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + const path = fileOf(root, 'run-st-0007'); + const original = readFileSync(path, 'utf8'); + writeFileSync(path, original.replace('"durationMs":1', '"durationMs":2')); + assert.equal(verifyRecord(readFileSync(path, 'utf8')).ok, true, 'the chain alone cannot see an edit to the last line'); + assert.deepEqual(store.verifyRun('run-st-0007').errors, [{ seq: 2, reason: 'head_mismatch' }]); + writeFileSync(path, original.split('\n').slice(0, 1).join('\n').concat('\n')); + assert.deepEqual(store.verifyRun('run-st-0007').errors.map((e) => e.reason), ['count_mismatch']); +})); + +test('a failed append breaks the run: later emits throw instead of chaining past an unknown tail', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0008' }); + run.emit('usage', {}); + chmodSync(fileOf(root, 'run-st-0008'), 0o400); + try { + assert.throws(() => run.emit('usage', {})); + chmodSync(fileOf(root, 'run-st-0008'), 0o600); + assert.throws(() => run.emit('usage', {}), /broken/); + } finally { chmodSync(fileOf(root, 'run-st-0008'), 0o600); } +})); + +test('task.done ends every grant bound to the task, in one grant.revoked written by the board', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0009', taskId: 'TASK-9', podId: 'pod-a', actor: AGENT }); + run.emit('grant.issued', { ...grantMeta('g-bound-1'), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'task.done' } }); + run.emit('grant.issued', { ...grantMeta('g-bound-2'), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'task.done' } }); + run.emit('grant.issued', { + ...grantMeta('g-timed', { expiresAt: '2026-10-25T23:00:00Z' }), binding: { taskId: 'TASK-9', podId: 'pod-a', endsOn: 'expiresAt' }, + }); + run.emit('grant.issued', grantMeta('g-free')); + run.emit('task.done', { doneBy: 'u-ben' }); + const events = store.readRun('run-st-0009'); + const [done, revoked] = events.slice(-2); + assert.equal(done.kind, 'task.done'); + assert.deepEqual(done.actor, { type: 'system', id: 'board' }); + assert.equal(revoked.kind, 'grant.revoked'); + assert.deepEqual(revoked.data, { grantIds: ['g-bound-1', 'g-bound-2'], reason: 'task.done' }); + assert.deepEqual(revoked.actor, { type: 'system', id: 'board' }); + assert.deepEqual(run.openGrants(), ['g-timed', 'g-free']); +})); + +test('revokeGrant: a lineage is one cascade entry; a lone grant is manual', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0010' }); + run.emit('grant.issued', grantMeta('g-root')); + run.emit('grant.issued', grantMeta('g-child', { parentGrantId: 'g-root' })); + run.emit('grant.issued', grantMeta('g-grandchild', { parentGrantId: 'g-child' })); + run.emit('grant.issued', grantMeta('g-lone')); + const cascade = run.revokeGrant('g-root', { actor: { type: 'human', id: 'u-ben' } }); + assert.deepEqual(cascade.data, { grantIds: ['g-root', 'g-child', 'g-grandchild'], reason: 'cascade' }); + assert.deepEqual(cascade.actor, { type: 'human', id: 'u-ben' }); + assert.deepEqual(run.revokeGrant('g-lone').data, { grantIds: ['g-lone'], reason: 'manual' }); + assert.equal(run.revokeGrant('g-lone'), null, 'an ended grant is not revoked twice'); + assert.deepEqual(run.openGrants(), []); +})); + +test('expireGrants records every grant past its expiresAt as expired', () => withStore((store) => { + const run = store.startRun({ runId: 'run-st-0011' }); + run.emit('grant.issued', grantMeta('g-old', { expiresAt: '2026-10-25T12:00:01Z' })); + run.emit('grant.issued', grantMeta('g-new', { expiresAt: '2026-10-26T00:00:00Z' })); + const entry = run.expireGrants(new Date('2026-10-25T13:00:00Z')); + assert.deepEqual(entry.data, { grantIds: ['g-old'], reason: 'expired' }); + assert.equal(run.expireGrants(new Date('2026-10-25T13:00:00Z')), null); +})); + +test('redact: known values never reach a line, even inside agent text', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-0012' }); + run.redact(['sk-secret-0123456789']); + run.emit('agent.message_chunk', { text: 'my key is sk-secret-0123456789, ok' }); + run.emit('tool.call', { toolCallId: 't', title: 'curl -H "x: sk-secret-0123456789"' }); + const raw = readFileSync(fileOf(root, 'run-st-0012'), 'utf8'); + assert.ok(!raw.includes('sk-secret-0123456789')); + assert.ok(raw.includes('[redacted]')); + assert.equal(store.verifyRun('run-st-0012').ok, true, 'scrubbing happens before hashing'); +})); diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js new file mode 100644 index 0000000000..a9458e00e2 --- /dev/null +++ b/packages/seats/test/record-valet.test.js @@ -0,0 +1,145 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; + +const FROM_ENV = 'sk-env-planted-0a1b2c3d4e5f'; +const FROM_FILE = 'sk-file-planted-6a7b8c9d0e1f'; +const FROM_RESOLVER = 'sk-resolver-planted-2f3e4d5c'; +const UNGRANTED = 'sk-ungranted-planted-9z8y7x6w'; +const OPTIONS = [ + { optionId: 'allow', kind: 'allow_once' }, { optionId: 'always', kind: 'allow_always' }, { optionId: 'reject', kind: 'reject_once' }, +]; + +const withRoot = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-record-valet-')); + try { return await fn(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const everyByte = (dir) => readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()).map((d) => readFileSync(join(d.parentPath ?? d.path, d.name), 'utf8')).join('\n'); + +test('no secret values in the record: env, file and resolver values, even echoed by the agent', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0001', taskId: 'TASK-1', podId: 'pod-a', actor: { type: 'agent', id: 'u-agent-1', owner: 'u-ana' } }); + const secretFile = join(root, 'token.txt'); + writeFileSync(secretFile, `${FROM_FILE}\n`, { mode: 0o600 }); + const grant = { + credentials: [ + { name: 'OPENAI_API_KEY', from: 'env' }, { name: 'FILE_TOKEN', from: 'file', ref: secretFile }, { name: 'BROKER_TOKEN', from: 'resolver' }, + ], + tools: { allow: ['read'], ask: ['fetch'] }, + }; + const env = await prepareRunEnv({ + grant, + baseEnv: { OPENAI_API_KEY: FROM_ENV, ANTHROPIC_API_KEY: UNGRANTED }, + resolveSecret: async () => FROM_RESOLVER, + root, + ctx: { trace: run }, + grantMeta: { + grantId: 'g-1', scope: { connectionId: 'conn-1', tools: ['sentry.get_issue'], audience: ['u-ana'] }, issuedBy: 'u-ana', taskId: 'TASK-1', podId: 'pod-a', + }, + }); + // Positive control: the values really reached the run, so their absence below means something. + assert.equal(env.env.OPENAI_API_KEY, FROM_ENV); + assert.equal(env.env.FILE_TOKEN, FROM_FILE); + assert.equal(env.env.BROKER_TOKEN, FROM_RESOLVER); + const decide = createPermissionPolicy(grant, { trace: run, approve: async () => ({ approved: true, by: 'u-ben', role: 'credential-owner' }) }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('agent.message_chunk', { text: `env ${FROM_ENV} file ${FROM_FILE} resolver ${FROM_RESOLVER}` }); + run.emit('tool.call', { toolCallId: 'tc-1', title: `curl -H "Authorization: Bearer ${FROM_RESOLVER}"`, kind: 'fetch' }); + await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch', kind: 'fetch' }, options: OPTIONS }); + run.emit('usage', { note: `${FROM_FILE}` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 3, attempts: 1 }); + await env.cleanup(); + + const bytes = everyByte(join(root, 'trace')); + assert.ok(bytes.includes('agent.message_chunk'), 'positive control: the agent text was written'); + for (const value of [FROM_ENV, FROM_FILE, FROM_RESOLVER, UNGRANTED]) assert.ok(!bytes.includes(value), `${value.slice(0, 12)}… reached the trace`); + assert.equal((bytes.match(/\[redacted\]/g) || []).length, 5); + const kinds = store.readRun('run-va-0001').map((e) => e.kind); + assert.deepEqual(kinds.slice(0, 2), ['grant.issued', 'credential.granted']); + assert.deepEqual(store.readRun('run-va-0001')[1].data, { names: ['OPENAI_API_KEY', 'FILE_TOKEN', 'BROKER_TOKEN'] }); + assert.equal(store.verifyRun('run-va-0001').ok, true); +})); + +test('grant.issued from grantMeta: bound to the task, ending on task.done by default', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0002', taskId: 'TASK-2', podId: 'pod-a' }); + const env = await prepareRunEnv({ + grant: { credentials: [] }, root, trace: run, + grantMeta: { + grantId: 'g-2', parentGrantId: 'g-root', issuedBy: 'u-ben', taskId: 'TASK-2', podId: 'pod-a', + scope: { connectionId: 'conn-ben', tools: ['github.get_pull_request'], writeMode: 'read', budget: { calls: 3 }, audience: ['u-ben'] }, + target: { kind: 'seat', id: 'u-agent-ben' }, + }, + }); + await env.cleanup(); + assert.equal(env.grantId, 'g-2'); + const [issued] = store.readRun('run-va-0002'); + assert.equal(issued.kind, 'grant.issued'); + assert.deepEqual(issued.data.binding, { taskId: 'TASK-2', podId: 'pod-a', endsOn: 'task.done' }); + assert.equal(issued.data.parentGrantId, 'g-root'); + assert.deepEqual(issued.actor, { type: 'human', id: 'u-ben' }); + run.emit('task.done', {}); + assert.deepEqual(store.readRun('run-va-0002').at(-1).data, { grantIds: ['g-2'], reason: 'task.done' }); +})); + +test('without grantMeta, prepareRunEnv emits what it always has', () => withRoot(async (root) => { + const events = []; + const env = await prepareRunEnv({ grant: { credentials: [] }, root, trace: { emit: (type, data) => events.push({ type, data }) } }); + await env.cleanup(); + assert.deepEqual(events, [{ type: 'credential.granted', data: { names: [] } }]); +})); + +const ask = async (answer) => { + const events = []; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approve: async () => answer, approveTimeoutMs: 1000, grantId: 'g-9', + }); + const result = await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + return { result, data: events[0].data }; +}; + +test('an approver approves only with true, or an object whose approved is exactly true', async () => { + for (const answer of [true, { approved: true }, { approved: true, by: 'u-ben' }]) { + assert.deepEqual((await ask(answer)).result, { outcome: 'selected', optionId: 'allow' }, JSON.stringify(answer)); + } + for (const answer of [{ approved: 'yes' }, { approved: 1 }, { approved: 'true' }, { approved: {} }, { approved: [true] }, 'yes', 1, {}, [true], null, undefined, false]) { + const { result, data } = await ask(answer); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }, `${JSON.stringify(answer)} must not approve`); + assert.equal(data.approval, 'rejected'); + } +}); + +test('the decision names who decided and as what, separate from the agent and its owner', async () => { + const { data } = await ask({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-1' }); + assert.equal(data.approval, 'approved'); + assert.equal(data.decidedBy, 'u-ben'); + assert.equal(data.decidedByRole, 'credential-owner'); + assert.equal(data.decisionCardId, 'card-1'); + assert.equal(data.grantId, 'g-9'); + const refused = await ask({ approved: false, by: 'u-ben', role: 'credential-owner' }); + assert.equal(refused.data.decidedBy, 'u-ben', 'a refusal is attributed too'); + // Malformed attribution is dropped rather than trusted. + const odd = await ask({ approved: true, by: 'Ben Smith', role: 'boss', decisionCardId: '' }); + assert.equal(odd.data.decidedBy, null); + assert.equal(odd.data.decidedByRole, null); + assert.equal('decisionCardId' in odd.data, false); + const bare = await ask(true); + assert.equal(bare.data.decidedBy, null, 'a bare true names nobody'); +}); + +test('no answer in time is decided by policy', async () => { + const events = []; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approve: () => new Promise(() => {}), approveTimeoutMs: 10, + }); + await decide({ toolCall: { toolCallId: 'tc-1', title: 'x', kind: 'fetch' }, options: OPTIONS }); + assert.equal(events[0].data.approval, 'timed_out'); + assert.equal(events[0].data.decidedBy, 'policy'); + assert.equal(events[0].data.decidedByRole, 'policy'); +}); From b7b71f63ae0b3a34d3084d3803056be4f5498a02 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:54:48 -0700 Subject: [PATCH 05/20] feat(seats): record v1 adds the ask, card rulings and an args digest Additive, so v stays 1. - permission.requested now carries `question` (agent text, scrubbed of known credential values like other agent text) and `askedOf: {id, role}`, both required, plus optional `grantId`, `decisionCardId` and `argsDigest`. The valet writes it when the host passes `askedOf`. A call a rule decides is asked of the grant's policy, and a malformed answer is recorded as the policy rather than as a person. - decision.ruled {decisionCardId, value, decidedBy, decidedByRole}: a card ruling not tied to a parked tool call, validated like permission.decided. - argsDigest on tool.call and the permission entries is "sha256:" + hex of the arguments' RFC 8785 form (argsDigestOf). Writers refuse any other value, so raw arguments can't ride in. - OTLP: a ruling joins the tool span whose request carried its card; seats.tool.args_digest goes on tool spans. - Doc: the revoke reasons are unchanged, and in a hosted pod server writers (TASK-513) supply expired and cascade. The sample's headline ask is tied to the grant and card-7f3, and Ana rules the retry on card-2b9. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- .../seats/docs/examples/TASK-101.otlp.json | 326 +++++++++++++++--- .../seats/docs/examples/run-ana-0001.jsonl | 39 ++- .../seats/docs/examples/run-ben-0001.jsonl | 24 +- packages/seats/docs/record-v1.md | 83 ++++- packages/seats/src/index.js | 2 +- packages/seats/src/record/format.js | 48 ++- packages/seats/src/record/otel.js | 10 +- packages/seats/src/trace/events.js | 5 +- packages/seats/src/valet/permissions.js | 59 +++- packages/seats/test-support/record-sample.js | 44 ++- packages/seats/test/record-example.test.js | 13 + packages/seats/test/record-format.test.js | 57 ++- packages/seats/test/record-otel.test.js | 26 +- packages/seats/test/record-valet.test.js | 61 ++++ 14 files changed, 675 insertions(+), 122 deletions(-) diff --git a/packages/seats/docs/examples/TASK-101.otlp.json b/packages/seats/docs/examples/TASK-101.otlp.json index 8d26baa11f..953d0ea88f 100644 --- a/packages/seats/docs/examples/TASK-101.otlp.json +++ b/packages/seats/docs/examples/TASK-101.otlp.json @@ -24,7 +24,7 @@ "name": "seats.run", "kind": 1, "startTimeUnixNano": "1792944002000000000", - "endTimeUnixNano": "1792944030000000000", + "endTimeUnixNano": "1792944031000000000", "attributes": [ { "key": "seats.run.id", @@ -273,7 +273,7 @@ }, { "timeUnixNano": "1792944012000000000", - "name": "liveness.recovery", + "name": "decision.ruled", "attributes": [ { "key": "seats.seq", @@ -281,6 +281,54 @@ "intValue": "11" } }, + { + "key": "seats.actor.type", + "value": { + "stringValue": "human" + } + }, + { + "key": "seats.actor.id", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.decided_by", + "value": { + "stringValue": "u-ana" + } + }, + { + "key": "seats.decided_by_role", + "value": { + "stringValue": "agent-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-2b9" + } + }, + { + "key": "seats.value", + "value": { + "stringValue": "retry" + } + } + ] + }, + { + "timeUnixNano": "1792944013000000000", + "name": "liveness.recovery", + "attributes": [ + { + "key": "seats.seq", + "value": { + "intValue": "12" + } + }, { "key": "seats.actor.type", "value": { @@ -308,13 +356,13 @@ ] }, { - "timeUnixNano": "1792944029000000000", + "timeUnixNano": "1792944030000000000", "name": "result.reported", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "28" + "intValue": "29" } }, { @@ -662,12 +710,12 @@ }, { "traceId": "d043bdec4346bb1253c06463a2c2cc13", - "spanId": "54966552492d24ee", + "spanId": "109cfe1b8a44b804", "parentSpanId": "4f7ea2299f50ef90", "name": "invoke_agent Ana's opencode", "kind": 1, - "startTimeUnixNano": "1792944013000000000", - "endTimeUnixNano": "1792944028000000000", + "startTimeUnixNano": "1792944014000000000", + "endTimeUnixNano": "1792944029000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -744,13 +792,13 @@ ], "events": [ { - "timeUnixNano": "1792944027000000000", + "timeUnixNano": "1792944028000000000", "name": "usage", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "26" + "intValue": "27" } }, { @@ -798,12 +846,12 @@ }, { "traceId": "d043bdec4346bb1253c06463a2c2cc13", - "spanId": "109cfe1b8a44b804", - "parentSpanId": "54966552492d24ee", + "spanId": "2f16f44fc7fa7d44", + "parentSpanId": "109cfe1b8a44b804", "name": "execute_tool git push origin fix/total-rounding", "kind": 1, - "startTimeUnixNano": "1792944014000000000", - "endTimeUnixNano": "1792944017000000000", + "startTimeUnixNano": "1792944015000000000", + "endTimeUnixNano": "1792944018000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -841,6 +889,12 @@ "stringValue": "native" } }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, { "key": "error.type", "value": { @@ -850,13 +904,13 @@ ], "events": [ { - "timeUnixNano": "1792944015000000000", + "timeUnixNano": "1792944016000000000", "name": "permission.requested", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "14" + "intValue": "15" } }, { @@ -883,6 +937,30 @@ "stringValue": "Ana's opencode" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, { "key": "seats.options", "value": { @@ -946,6 +1024,12 @@ } } }, + { + "key": "seats.question", + "value": { + "stringValue": "git push origin fix/total-rounding" + } + }, { "key": "seats.tool_call_id", "value": { @@ -955,13 +1039,13 @@ ] }, { - "timeUnixNano": "1792944016000000000", + "timeUnixNano": "1792944017000000000", "name": "permission.decided", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "15" + "intValue": "16" } }, { @@ -976,6 +1060,12 @@ "stringValue": "grant:grant-ana-sentry-1" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92" + } + }, { "key": "seats.decided_by", "value": { @@ -1033,13 +1123,13 @@ ] }, { - "timeUnixNano": "1792944017000000000", + "timeUnixNano": "1792944018000000000", "name": "tool.update", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "16" + "intValue": "17" } }, { @@ -1088,12 +1178,12 @@ }, { "traceId": "d043bdec4346bb1253c06463a2c2cc13", - "spanId": "b3182723ae254b59", - "parentSpanId": "54966552492d24ee", + "spanId": "e9bb6dcc7812bd2c", + "parentSpanId": "109cfe1b8a44b804", "name": "execute_tool sentry.resolve_issue", "kind": 1, - "startTimeUnixNano": "1792944018000000000", - "endTimeUnixNano": "1792944021000000000", + "startTimeUnixNano": "1792944019000000000", + "endTimeUnixNano": "1792944022000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -1130,17 +1220,23 @@ "value": { "stringValue": "connector" } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } } ], "events": [ { - "timeUnixNano": "1792944019000000000", + "timeUnixNano": "1792944020000000000", "name": "permission.requested", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "18" + "intValue": "19" } }, { @@ -1167,6 +1263,36 @@ "stringValue": "Ana's opencode" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "u-ben" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "credential-owner" + } + }, + { + "key": "seats.decision_card_id", + "value": { + "stringValue": "card-7f3" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, { "key": "seats.options", "value": { @@ -1230,6 +1356,12 @@ } } }, + { + "key": "seats.question", + "value": { + "stringValue": "Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212" + } + }, { "key": "seats.tool_call_id", "value": { @@ -1239,13 +1371,13 @@ ] }, { - "timeUnixNano": "1792944020000000000", + "timeUnixNano": "1792944021000000000", "name": "permission.decided", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "19" + "intValue": "20" } }, { @@ -1266,6 +1398,12 @@ "stringValue": "approved" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1" + } + }, { "key": "seats.decided_by", "value": { @@ -1329,13 +1467,13 @@ ] }, { - "timeUnixNano": "1792944021000000000", + "timeUnixNano": "1792944022000000000", "name": "tool.update", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "20" + "intValue": "21" } }, { @@ -1383,12 +1521,12 @@ }, { "traceId": "d043bdec4346bb1253c06463a2c2cc13", - "spanId": "400ad84d51bc8edf", - "parentSpanId": "54966552492d24ee", + "spanId": "d54dc8e7e5d85bf5", + "parentSpanId": "109cfe1b8a44b804", "name": "execute_tool commonly_post_message", "kind": 1, - "startTimeUnixNano": "1792944022000000000", - "endTimeUnixNano": "1792944025000000000", + "startTimeUnixNano": "1792944023000000000", + "endTimeUnixNano": "1792944026000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -1425,17 +1563,23 @@ "value": { "stringValue": "commonly" } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } } ], "events": [ { - "timeUnixNano": "1792944023000000000", + "timeUnixNano": "1792944024000000000", "name": "permission.requested", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "22" + "intValue": "23" } }, { @@ -1462,6 +1606,30 @@ "stringValue": "Ana's opencode" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ana-sentry-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ana-sentry-1" + } + }, { "key": "seats.options", "value": { @@ -1525,6 +1693,12 @@ } } }, + { + "key": "seats.question", + "value": { + "stringValue": "commonly_post_message: the fix summary, to the pod" + } + }, { "key": "seats.tool_call_id", "value": { @@ -1534,13 +1708,13 @@ ] }, { - "timeUnixNano": "1792944024000000000", + "timeUnixNano": "1792944025000000000", "name": "permission.decided", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "23" + "intValue": "24" } }, { @@ -1555,6 +1729,12 @@ "stringValue": "grant:grant-ana-sentry-1" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5" + } + }, { "key": "seats.decided_by", "value": { @@ -1612,13 +1792,13 @@ ] }, { - "timeUnixNano": "1792944025000000000", + "timeUnixNano": "1792944026000000000", "name": "tool.update", "attributes": [ { "key": "seats.seq", "value": { - "intValue": "24" + "intValue": "25" } }, { @@ -1669,8 +1849,8 @@ "spanId": "a4aa63e7490207ea", "name": "seats.run", "kind": 1, - "startTimeUnixNano": "1792944032000000000", - "endTimeUnixNano": "1792944043000000000", + "startTimeUnixNano": "1792944033000000000", + "endTimeUnixNano": "1792944044000000000", "attributes": [ { "key": "seats.run.id", @@ -1705,7 +1885,7 @@ ], "events": [ { - "timeUnixNano": "1792944032000000000", + "timeUnixNano": "1792944033000000000", "name": "grant.issued", "attributes": [ { @@ -1813,7 +1993,7 @@ ] }, { - "timeUnixNano": "1792944033000000000", + "timeUnixNano": "1792944034000000000", "name": "credential.granted", "attributes": [ { @@ -1849,7 +2029,7 @@ ] }, { - "timeUnixNano": "1792944041000000000", + "timeUnixNano": "1792944042000000000", "name": "task.done", "attributes": [ { @@ -1879,7 +2059,7 @@ ] }, { - "timeUnixNano": "1792944042000000000", + "timeUnixNano": "1792944043000000000", "name": "grant.revoked", "attributes": [ { @@ -1931,8 +2111,8 @@ "parentSpanId": "a4aa63e7490207ea", "name": "invoke_agent Ben's goose", "kind": 1, - "startTimeUnixNano": "1792944034000000000", - "endTimeUnixNano": "1792944040000000000", + "startTimeUnixNano": "1792944035000000000", + "endTimeUnixNano": "1792944041000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -2000,8 +2180,8 @@ "parentSpanId": "5fd2950fcbddd334", "name": "execute_tool github.get_pull_request", "kind": 1, - "startTimeUnixNano": "1792944035000000000", - "endTimeUnixNano": "1792944038000000000", + "startTimeUnixNano": "1792944036000000000", + "endTimeUnixNano": "1792944039000000000", "attributes": [ { "key": "gen_ai.operation.name", @@ -2038,11 +2218,17 @@ "value": { "stringValue": "connector" } + }, + { + "key": "seats.tool.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } } ], "events": [ { - "timeUnixNano": "1792944036000000000", + "timeUnixNano": "1792944037000000000", "name": "permission.requested", "attributes": [ { @@ -2075,6 +2261,30 @@ "stringValue": "Ben's goose" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } + }, + { + "key": "seats.asked_of.id", + "value": { + "stringValue": "grant:grant-ben-review-1" + } + }, + { + "key": "seats.asked_of.role", + "value": { + "stringValue": "policy" + } + }, + { + "key": "seats.grant_id", + "value": { + "stringValue": "grant-ben-review-1" + } + }, { "key": "seats.options", "value": { @@ -2138,6 +2348,12 @@ } } }, + { + "key": "seats.question", + "value": { + "stringValue": "Read pull request #212" + } + }, { "key": "seats.tool_call_id", "value": { @@ -2147,7 +2363,7 @@ ] }, { - "timeUnixNano": "1792944037000000000", + "timeUnixNano": "1792944038000000000", "name": "permission.decided", "attributes": [ { @@ -2168,6 +2384,12 @@ "stringValue": "grant:grant-ben-review-1" } }, + { + "key": "seats.args_digest", + "value": { + "stringValue": "sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee" + } + }, { "key": "seats.decided_by", "value": { @@ -2225,7 +2447,7 @@ ] }, { - "timeUnixNano": "1792944038000000000", + "timeUnixNano": "1792944039000000000", "name": "tool.update", "attributes": [ { diff --git a/packages/seats/docs/examples/run-ana-0001.jsonl b/packages/seats/docs/examples/run-ana-0001.jsonl index 902db3a75a..05afbb7280 100644 --- a/packages/seats/docs/examples/run-ana-0001.jsonl +++ b/packages/seats/docs/examples/run-ana-0001.jsonl @@ -8,22 +8,23 @@ {"actor":{"id":"seats","type":"system"},"data":{"idleMs":90000,"windowMs":90000},"kind":"liveness.stalled","podId":"pod-checkout","prev":"212b5d465b8936032324127c8b834523df956bc6bc5fde15004d0b4932009dfb","runId":"run-ana-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:09.000Z","v":1} {"actor":{"id":"seats","type":"system"},"data":{"action":"cancel","attempt":1},"kind":"liveness.recovery","podId":"pod-checkout","prev":"260faaf14f1c7bf121338f251a1c88e41c78689ccdbfa0fe3f36831bda5c3848","runId":"run-ana-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:10.000Z","v":1} {"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"failure":{"detail":"no progress signal for 90000ms (window 90000ms)","kind":"stuck"},"outcome":"stuck"},"kind":"attempt.settled","podId":"pod-checkout","prev":"c7637334d2a2bdeea8d5d6ae3b402d6d348bcd18ee563cbe2ef00576cb7c44ad","runId":"run-ana-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:11.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"action":"retry","attempt":2},"kind":"liveness.recovery","podId":"pod-checkout","prev":"0ee1be8a38ec355ebd500c99c09517e2607ab01a61ef6989770bb09a1b1f3adf","runId":"run-ana-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:12.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":2,"cli":{"name":"opencode","version":"1.19.0"},"sessionId":"ses-ana-2"},"kind":"attempt.started","podId":"pod-checkout","prev":"222b2d342255178de812757d277bf1913c14ee21178b7f67da8c2ddc1342ed42","runId":"run-ana-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:13.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"execute","status":"pending","surface":"native","title":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"tool.call","podId":"pod-checkout","prev":"ad5d76d6df5f3aa0a6c914602248135ff230fb75f3abc79cace9ff5094b257d0","runId":"run-ana-0001","schema":"seats.record","seq":13,"taskId":"TASK-101","ts":"2026-10-25T16:00:14.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-2"},"kind":"permission.requested","podId":"pod-checkout","prev":"34c7eb2c6facb2bb29bef8c63443eac1179a1b3e80bf26d4425df090b686303a","runId":"run-ana-0001","schema":"seats.record","seq":14,"taskId":"TASK-101","ts":"2026-10-25T16:00:15.000Z","v":1} -{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"reject_once","optionId":"reject","outcome":"selected","rule":{"list":"deny","pattern":"execute:git push*"},"toolCallId":"tc-2"},"kind":"permission.decided","podId":"pod-checkout","prev":"c51c441f24bc1126084d5918d180bb0735c9aab07b3756426cdf274333c38193","runId":"run-ana-0001","schema":"seats.record","seq":15,"taskId":"TASK-101","ts":"2026-10-25T16:00:16.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"failed","toolCallId":"tc-2"},"kind":"tool.update","podId":"pod-checkout","prev":"46f1eeb72be62623c2b8605a6115370207f11ccf1edccfc429cfe641edd37e50","runId":"run-ana-0001","schema":"seats.record","seq":16,"taskId":"TASK-101","ts":"2026-10-25T16:00:17.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"fetch","name":"sentry.resolve_issue","status":"pending","surface":"connector","title":"sentry.resolve_issue CHECKOUT-88","toolCallId":"tc-3"},"kind":"tool.call","podId":"pod-checkout","prev":"6f59e269f201abfa36dc338fe094a287fcee110cf734df26b8072db46a7476fe","runId":"run-ana-0001","schema":"seats.record","seq":17,"taskId":"TASK-101","ts":"2026-10-25T16:00:18.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-3"},"kind":"permission.requested","podId":"pod-checkout","prev":"259adaed1cdf12c494fd150cb604a54f51b7780154f87eba8ad40d2796d1a1d0","runId":"run-ana-0001","schema":"seats.record","seq":18,"taskId":"TASK-101","ts":"2026-10-25T16:00:19.000Z","v":1} -{"actor":{"id":"u-ben","type":"human"},"data":{"approval":"approved","decidedBy":"u-ben","decidedByRole":"credential-owner","decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"ask","pattern":"fetch"},"toolCallId":"tc-3"},"kind":"permission.decided","podId":"pod-checkout","prev":"bbf582d7342a8966424e1ff04805088c8468f2ac5d55dfc71c5f714bd6d183cc","runId":"run-ana-0001","schema":"seats.record","seq":19,"taskId":"TASK-101","ts":"2026-10-25T16:00:20.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-3"},"kind":"tool.update","podId":"pod-checkout","prev":"9e2395f74135669ef532fcb750602ca8617803ac8d061a1b5680a430be1ba687","runId":"run-ana-0001","schema":"seats.record","seq":20,"taskId":"TASK-101","ts":"2026-10-25T16:00:21.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"kind":"other","name":"commonly_post_message","status":"pending","surface":"commonly","title":"commonly_post_message","toolCallId":"tc-4"},"kind":"tool.call","podId":"pod-checkout","prev":"3257330abfb7e0e9e9be4bc549df79e7da6c3d154e6b4300bbba9abb965fd976","runId":"run-ana-0001","schema":"seats.record","seq":21,"taskId":"TASK-101","ts":"2026-10-25T16:00:22.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-4"},"kind":"permission.requested","podId":"pod-checkout","prev":"e3d66ce0c7ab8dd4bf87583f2bb15d1d8a3ff1b84a12820f975041f65114aec7","runId":"run-ana-0001","schema":"seats.record","seq":22,"taskId":"TASK-101","ts":"2026-10-25T16:00:23.000Z","v":1} -{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"other:commonly_*"},"toolCallId":"tc-4"},"kind":"permission.decided","podId":"pod-checkout","prev":"973c505153298862657039b572aeb0cb27bc86030d3dd220ad146437f7689da1","runId":"run-ana-0001","schema":"seats.record","seq":23,"taskId":"TASK-101","ts":"2026-10-25T16:00:24.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-4"},"kind":"tool.update","podId":"pod-checkout","prev":"d9a5a1169bd53ad6c60bf10e3cb7ef1a7a36a93b4a5b2c9ef442df999d52825d","runId":"run-ana-0001","schema":"seats.record","seq":24,"taskId":"TASK-101","ts":"2026-10-25T16:00:25.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"text":"Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"f4fa58c0c29f4c988950042cc5c42b25355ec0cb00b9f6bf3abc5011cd63e88f","runId":"run-ana-0001","schema":"seats.record","seq":25,"taskId":"TASK-101","ts":"2026-10-25T16:00:26.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"inputTokens":18234,"outputTokens":2210},"kind":"usage","podId":"pod-checkout","prev":"12364a5e040c116f6acbceb87cea8c712853c5b4bada8680fffab671eeaf223b","runId":"run-ana-0001","schema":"seats.record","seq":26,"taskId":"TASK-101","ts":"2026-10-25T16:00:27.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"attempt":2,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"24b24a0f64242aacf7b78097b131eba62c469d35c2125453136df6c5e87853eb","runId":"run-ana-0001","schema":"seats.record","seq":27,"taskId":"TASK-101","ts":"2026-10-25T16:00:28.000Z","v":1} -{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"refs":[{"id":"#212","type":"pr"}],"status":"done"},"kind":"result.reported","podId":"pod-checkout","prev":"caad0b53230898f529c5f0c332a12763193c7786c190603cd7774fa5db59ed1e","runId":"run-ana-0001","schema":"seats.record","seq":28,"taskId":"TASK-101","ts":"2026-10-25T16:00:29.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"attempts":2,"durationMs":412000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"c3c79cddb97be9fcbd02bcaf8081b23026c78caa9d3810c73f79d1c9b79b52fb","runId":"run-ana-0001","schema":"seats.record","seq":29,"taskId":"TASK-101","ts":"2026-10-25T16:00:30.000Z","v":1} +{"actor":{"id":"u-ana","type":"human"},"data":{"decidedBy":"u-ana","decidedByRole":"agent-owner","decisionCardId":"card-2b9","value":"retry"},"kind":"decision.ruled","podId":"pod-checkout","prev":"0ee1be8a38ec355ebd500c99c09517e2607ab01a61ef6989770bb09a1b1f3adf","runId":"run-ana-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:12.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"action":"retry","attempt":2},"kind":"liveness.recovery","podId":"pod-checkout","prev":"440445ac0d77d87d42c2bcb22b5efa55331e923f0e36b3447dd9a9a64ece95b2","runId":"run-ana-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:13.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"adapter":"opencode","attempt":2,"cli":{"name":"opencode","version":"1.19.0"},"sessionId":"ses-ana-2"},"kind":"attempt.started","podId":"pod-checkout","prev":"463755acfb130eda7a974c6be8d4af9a080556d5e103ecd16d90bdb77e2e7f43","runId":"run-ana-0001","schema":"seats.record","seq":13,"taskId":"TASK-101","ts":"2026-10-25T16:00:14.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","kind":"execute","status":"pending","surface":"native","title":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"tool.call","podId":"pod-checkout","prev":"8370cccac6497cee039bda506eaabbaaa6e81c3067520062d6bb91329853e41e","runId":"run-ana-0001","schema":"seats.record","seq":14,"taskId":"TASK-101","ts":"2026-10-25T16:00:15.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","askedOf":{"id":"grant:grant-ana-sentry-1","role":"policy"},"grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"git push origin fix/total-rounding","toolCallId":"tc-2"},"kind":"permission.requested","podId":"pod-checkout","prev":"ad5435f5ed0c1d500e8c2ddaec7b8023d72660d9daea5f5c8c57266fc48c8e63","runId":"run-ana-0001","schema":"seats.record","seq":15,"taskId":"TASK-101","ts":"2026-10-25T16:00:16.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"argsDigest":"sha256:fca8e99d0cd7188c5fd43bd7cd79272d5598afb370cb5aac7b7ccb151087bd92","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"reject_once","optionId":"reject","outcome":"selected","rule":{"list":"deny","pattern":"execute:git push*"},"toolCallId":"tc-2"},"kind":"permission.decided","podId":"pod-checkout","prev":"bae28e9f22669c2f9b24668500e0faa57d193a936dc3240a6e2f655751180d20","runId":"run-ana-0001","schema":"seats.record","seq":16,"taskId":"TASK-101","ts":"2026-10-25T16:00:17.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"failed","toolCallId":"tc-2"},"kind":"tool.update","podId":"pod-checkout","prev":"d41a409cfd72534a80e4d08609daafbd0440a22c4964631ff99c49fbd4d13475","runId":"run-ana-0001","schema":"seats.record","seq":17,"taskId":"TASK-101","ts":"2026-10-25T16:00:18.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","kind":"fetch","name":"sentry.resolve_issue","status":"pending","surface":"connector","title":"Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212","toolCallId":"tc-3"},"kind":"tool.call","podId":"pod-checkout","prev":"bce5d5620503b5040aebc590e429962fd453ac6bb84648b7a3e725677749724f","runId":"run-ana-0001","schema":"seats.record","seq":18,"taskId":"TASK-101","ts":"2026-10-25T16:00:19.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","askedOf":{"id":"u-ben","role":"credential-owner"},"decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212","toolCallId":"tc-3"},"kind":"permission.requested","podId":"pod-checkout","prev":"cfa49b7b5892c724099e22752af46b6b6f78040886dc404019bcfedb48605206","runId":"run-ana-0001","schema":"seats.record","seq":19,"taskId":"TASK-101","ts":"2026-10-25T16:00:20.000Z","v":1} +{"actor":{"id":"u-ben","type":"human"},"data":{"approval":"approved","argsDigest":"sha256:70bf019541b1a36a99c763ca15e36c24c15cbe7880b643d897ebd40ff3deeff1","decidedBy":"u-ben","decidedByRole":"credential-owner","decisionCardId":"card-7f3","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"ask","pattern":"fetch"},"toolCallId":"tc-3"},"kind":"permission.decided","podId":"pod-checkout","prev":"e8b32872b12d59434da0832df0c96cc1d7c3fc5ca9cfc14612cb00c5af624b62","runId":"run-ana-0001","schema":"seats.record","seq":20,"taskId":"TASK-101","ts":"2026-10-25T16:00:21.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-3"},"kind":"tool.update","podId":"pod-checkout","prev":"2a277938e8fd32717316cc03f6f79d46034b405b3bf729157b80bb7852bf14f6","runId":"run-ana-0001","schema":"seats.record","seq":21,"taskId":"TASK-101","ts":"2026-10-25T16:00:22.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","kind":"other","name":"commonly_post_message","status":"pending","surface":"commonly","title":"commonly_post_message: the fix summary, to the pod","toolCallId":"tc-4"},"kind":"tool.call","podId":"pod-checkout","prev":"fc2b15b652bcb8f00463f829ba89a236220c55f8630db6f6d2b32834bd7ba25e","runId":"run-ana-0001","schema":"seats.record","seq":22,"taskId":"TASK-101","ts":"2026-10-25T16:00:23.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","askedOf":{"id":"grant:grant-ana-sentry-1","role":"policy"},"grantId":"grant-ana-sentry-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"commonly_post_message: the fix summary, to the pod","toolCallId":"tc-4"},"kind":"permission.requested","podId":"pod-checkout","prev":"23550e8771e7124cf2ffb378bf23cb015fc909da9ab30b38c71ab3d72b636379","runId":"run-ana-0001","schema":"seats.record","seq":23,"taskId":"TASK-101","ts":"2026-10-25T16:00:24.000Z","v":1} +{"actor":{"id":"grant:grant-ana-sentry-1","type":"policy"},"data":{"argsDigest":"sha256:f3e2b6ca902dc5a1ab8afac954c241c4dc1034a481fc6bede1ee7a6bc33dcee5","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ana-sentry-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"other:commonly_*"},"toolCallId":"tc-4"},"kind":"permission.decided","podId":"pod-checkout","prev":"7036a70b913d9b4c6e2a8cdcd965be61f3881b2c1fe8f74617d5b45c44a29e67","runId":"run-ana-0001","schema":"seats.record","seq":24,"taskId":"TASK-101","ts":"2026-10-25T16:00:25.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"status":"completed","toolCallId":"tc-4"},"kind":"tool.update","podId":"pod-checkout","prev":"08800755ccfa37befaf6d6eeb849d79e97514f1f8fdfa9a68594487e694da893","runId":"run-ana-0001","schema":"seats.record","seq":25,"taskId":"TASK-101","ts":"2026-10-25T16:00:26.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"text":"Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"f63d36617240b79419e99a85bd1d5376b3cc4d7c6a7670abc5bb9a57308a200f","runId":"run-ana-0001","schema":"seats.record","seq":26,"taskId":"TASK-101","ts":"2026-10-25T16:00:27.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"inputTokens":18234,"outputTokens":2210},"kind":"usage","podId":"pod-checkout","prev":"99a5c2a6d12f37d7ea10bc4aa0780f5ea3edcaeb10fed13757c8969f31364c6a","runId":"run-ana-0001","schema":"seats.record","seq":27,"taskId":"TASK-101","ts":"2026-10-25T16:00:28.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":2,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"a53827fe95252e39d8911788de097522424b66db9cd9ad717721c7ca1be7cb38","runId":"run-ana-0001","schema":"seats.record","seq":28,"taskId":"TASK-101","ts":"2026-10-25T16:00:29.000Z","v":1} +{"actor":{"id":"u-agent-ana-opencode","name":"Ana's opencode","owner":"u-ana","type":"agent"},"data":{"refs":[{"id":"#212","type":"pr"}],"status":"done"},"kind":"result.reported","podId":"pod-checkout","prev":"1daff17c56410a04940183bfb8a19272ec9162aa5b53e8982a8badf6c51000f0","runId":"run-ana-0001","schema":"seats.record","seq":29,"taskId":"TASK-101","ts":"2026-10-25T16:00:30.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":2,"durationMs":412000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"4e47933b81d799b01bac98cb915f0b9d46092a62295f4f1dd62389282a3022e6","runId":"run-ana-0001","schema":"seats.record","seq":30,"taskId":"TASK-101","ts":"2026-10-25T16:00:31.000Z","v":1} diff --git a/packages/seats/docs/examples/run-ben-0001.jsonl b/packages/seats/docs/examples/run-ben-0001.jsonl index 60ad22d6c3..b6fd8ce3e9 100644 --- a/packages/seats/docs/examples/run-ben-0001.jsonl +++ b/packages/seats/docs/examples/run-ben-0001.jsonl @@ -1,12 +1,12 @@ -{"actor":{"id":"u-ben","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ben-review-1","issuedBy":"u-ben","scope":{"audience":["u-ben"],"connectionId":"conn-ben-github","tools":["github.get_pull_request"],"writeMode":"read"},"target":{"id":"u-agent-ben-goose","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":null,"runId":"run-ben-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:32.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENAI_API_KEY"]},"kind":"credential.granted","podId":"pod-checkout","prev":"f1ad10d77360347b5a51560adb0aad2d3b2dd8dcb872fb962b83e06f1735e2d4","runId":"run-ben-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:33.000Z","v":1} -{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"adapter":"goose","attempt":1,"cli":{"name":"goose","version":"1.54.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"54299a711ac2afca451e8be0574218f91f02ce64041b1097f8cd29f3e169ce5f","runId":"run-ben-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:34.000Z","v":1} -{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"kind":"read","name":"github.get_pull_request","status":"pending","surface":"connector","title":"github.get_pull_request #212","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"16c619cb9a9fb8f760186168a22685f2ec6708948a9ff70e35294140921dcccf","runId":"run-ben-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:35.000Z","v":1} -{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"toolCallId":"tc-1"},"kind":"permission.requested","podId":"pod-checkout","prev":"77ee7b8804e3d60b7d34050b10df34bd8b0d8ce85a4cbdc48bc57d6c099edf40","runId":"run-ben-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:36.000Z","v":1} -{"actor":{"id":"grant:grant-ben-review-1","type":"policy"},"data":{"decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ben-review-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"read"},"toolCallId":"tc-1"},"kind":"permission.decided","podId":"pod-checkout","prev":"dec1718f26e8617628e7f3c522ab23a6a4601802920b2723c2cae700be3de62a","runId":"run-ben-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:37.000Z","v":1} -{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"a00470b2fb1d368d5210df55e829815c8692a546347731e033687aca5396826d","runId":"run-ben-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:38.000Z","v":1} -{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"text":"Reviewed #212: the rounding fix is right and the half-cent case is tested."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"8ab27c571e0f78429a09a77d01d2ff63ec2052503b0d0a49207056719f416949","runId":"run-ben-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:39.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"2df713d5685675553f11676224b674459da8cf14b389879c9e4ec35a203768b5","runId":"run-ben-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:40.000Z","v":1} -{"actor":{"id":"board","type":"system"},"data":{"doneBy":"u-ben"},"kind":"task.done","podId":"pod-checkout","prev":"029be6e1a3f91e8309ef92e4c5faf6c89fb22eb8fb087f8ef4b11f6a45167803","runId":"run-ben-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:41.000Z","v":1} -{"actor":{"id":"board","type":"system"},"data":{"grantIds":["grant-ben-review-1"],"reason":"task.done"},"kind":"grant.revoked","podId":"pod-checkout","prev":"647595a7d6a12380b8b004caefb6ac2f48c7f287fe70f8a0e6862f85dffa14f1","runId":"run-ben-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:42.000Z","v":1} -{"actor":{"id":"seats","type":"system"},"data":{"attempts":1,"durationMs":95000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"c631427d4c505347aeacc1135eeab6519eaf8fdec9d9302c46c79aa78c6795de","runId":"run-ben-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:43.000Z","v":1} +{"actor":{"id":"u-ben","type":"human"},"data":{"binding":{"endsOn":"task.done","podId":"pod-checkout","taskId":"TASK-101"},"expiresAt":"2026-10-25T20:00:00.000Z","grantId":"grant-ben-review-1","issuedBy":"u-ben","scope":{"audience":["u-ben"],"connectionId":"conn-ben-github","tools":["github.get_pull_request"],"writeMode":"read"},"target":{"id":"u-agent-ben-goose","kind":"seat"}},"kind":"grant.issued","podId":"pod-checkout","prev":null,"runId":"run-ben-0001","schema":"seats.record","seq":1,"taskId":"TASK-101","ts":"2026-10-25T16:00:33.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"names":["OPENAI_API_KEY"]},"kind":"credential.granted","podId":"pod-checkout","prev":"ab5416e866a17b4fa875a4127be03bd14d2294d9a5f1271b84fb8dd4fa6abfff","runId":"run-ben-0001","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:00:34.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"adapter":"goose","attempt":1,"cli":{"name":"goose","version":"1.54.0"}},"kind":"attempt.started","podId":"pod-checkout","prev":"6b2d6a9397ea15e894dc3f580acf90af4326073a8b3267ef869be8d04dcc8c97","runId":"run-ben-0001","schema":"seats.record","seq":3,"taskId":"TASK-101","ts":"2026-10-25T16:00:35.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","kind":"read","name":"github.get_pull_request","status":"pending","surface":"connector","title":"Read pull request #212","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"2e671132e0b51bea5f40b804e58badc66082619d6da280c9538f12ec8d0ae38a","runId":"run-ben-0001","schema":"seats.record","seq":4,"taskId":"TASK-101","ts":"2026-10-25T16:00:36.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","askedOf":{"id":"grant:grant-ben-review-1","role":"policy"},"grantId":"grant-ben-review-1","options":[{"kind":"allow_once","optionId":"allow"},{"kind":"allow_always","optionId":"always"},{"kind":"reject_once","optionId":"reject"}],"question":"Read pull request #212","toolCallId":"tc-1"},"kind":"permission.requested","podId":"pod-checkout","prev":"647dfa8b21dba1911c740b2eb0ca1a2f38eade8adedf53ab54f44999701ae38c","runId":"run-ben-0001","schema":"seats.record","seq":5,"taskId":"TASK-101","ts":"2026-10-25T16:00:37.000Z","v":1} +{"actor":{"id":"grant:grant-ben-review-1","type":"policy"},"data":{"argsDigest":"sha256:49be40400a494e11be85f05dfaa069f693cbcccf50b9d86674f869586a09b9ee","decidedBy":"policy","decidedByRole":"policy","grantId":"grant-ben-review-1","kind":"allow_once","optionId":"allow","outcome":"selected","rule":{"list":"allow","pattern":"read"},"toolCallId":"tc-1"},"kind":"permission.decided","podId":"pod-checkout","prev":"36f32b99f8e1e72f616a5e1bccb4190b150e5d45301066f9fbd4a07342ff36f5","runId":"run-ben-0001","schema":"seats.record","seq":6,"taskId":"TASK-101","ts":"2026-10-25T16:00:38.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"status":"completed","toolCallId":"tc-1"},"kind":"tool.update","podId":"pod-checkout","prev":"1b5374fb6b7d6157ab2baa137b9294f1df5ee4d1018ff0ffbaff5cb54207d044","runId":"run-ben-0001","schema":"seats.record","seq":7,"taskId":"TASK-101","ts":"2026-10-25T16:00:39.000Z","v":1} +{"actor":{"id":"u-agent-ben-goose","name":"Ben's goose","owner":"u-ben","type":"agent"},"data":{"text":"Reviewed #212: the rounding fix is right and the half-cent case is tested."},"kind":"agent.message_chunk","podId":"pod-checkout","prev":"75b890cf31a88f3e147662c84844f3c7054554ef2ad8cc3b7855ab93d7a18405","runId":"run-ben-0001","schema":"seats.record","seq":8,"taskId":"TASK-101","ts":"2026-10-25T16:00:40.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempt":1,"outcome":"ok"},"kind":"attempt.settled","podId":"pod-checkout","prev":"6bcb6d2e835bebdd2bd2e124ee9f124518347642cebf6fc645c4e57e278120b9","runId":"run-ben-0001","schema":"seats.record","seq":9,"taskId":"TASK-101","ts":"2026-10-25T16:00:41.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"doneBy":"u-ben"},"kind":"task.done","podId":"pod-checkout","prev":"a24cfa42f1edfff156f029724d40e84ef51874ef3d311920c824023e42ff528e","runId":"run-ben-0001","schema":"seats.record","seq":10,"taskId":"TASK-101","ts":"2026-10-25T16:00:42.000Z","v":1} +{"actor":{"id":"board","type":"system"},"data":{"grantIds":["grant-ben-review-1"],"reason":"task.done"},"kind":"grant.revoked","podId":"pod-checkout","prev":"2b9a41ec20f94c8da8fe6bab7e0c90ba4002aa3d36fb167f30ca6286e3bc66b8","runId":"run-ben-0001","schema":"seats.record","seq":11,"taskId":"TASK-101","ts":"2026-10-25T16:00:43.000Z","v":1} +{"actor":{"id":"seats","type":"system"},"data":{"attempts":1,"durationMs":95000,"outcome":"ok"},"kind":"run.settled","podId":"pod-checkout","prev":"0518c6a4515f9cac06b19169c84cd897c3dbc81757327c25e251cbf56f5a2ffa","runId":"run-ben-0001","schema":"seats.record","seq":12,"taskId":"TASK-101","ts":"2026-10-25T16:00:44.000Z","v":1} diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 879fde3db8..b0b395f910 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -140,6 +140,11 @@ one of: rest are its descendants, all ended by the same write; - `manual`: a person or policy revoked exactly these grants. The actor says who. +The reasons are unchanged by the 2026-10-25 revision. In a hosted pod, server +writers (TASK-513) supply `expired` and `cascade`, because the server holds +grant expiry and lineage. The local store writes the same two reasons from +`expireGrants` and `revokeGrant` for a run that has no server writer. + **What revocation means in v1.** A revoked grant is no longer valid. A credential that a run placed in its agent process's environment stays there until that process exits. A revocation records that the grant's term ended; @@ -149,8 +154,22 @@ it does not prove the process lost access. That happens at `run.settled`. | kind | data | default actor | |---|---|---| -| `permission.requested` | `{toolCallId?, options: [{optionId, kind}]}` | run | -| `permission.decided` | `{outcome, optionId?, kind?, toolCallId, rule, approval?, decidedBy, decidedByRole, decisionCardId?, grantId?}` | the decider: a human actor for a user id, a policy actor for `policy` | +| `permission.requested` | `{toolCallId?, options: [{optionId, kind}], question, askedOf: {id, role}, grantId?, decisionCardId?, argsDigest?}` | run | +| `permission.decided` | `{outcome, optionId?, kind?, toolCallId, rule, approval?, decidedBy, decidedByRole, decisionCardId?, grantId?, argsDigest?}` | the decider: a human actor for a user id, a policy actor for `policy` | +| `decision.ruled` | `{decisionCardId, value, decidedBy, decidedByRole}` | the decider, as for `permission.decided` | + +**The request:** +- `question` is required: the human-readable ask, normally the tool call's + title. It is agent text, so it is scrubbed of known credential values + before it is written, like other agent text (section 3, rule 1). +- `askedOf` is required: `{id, role}`. + - `id` is an id token, under the same rule as `actor.id`. + - `role` is from the `decidedByRole` enum below. + - A call that a rule decides, with no person asked, is asked of the policy: + `{id: "grant:<grantId>", role: "policy"}`. +- `grantId` is the grant the call would act under. +- `decisionCardId` is the card the ask rides on, when there is one. +- `argsDigest` identifies the call's arguments (under Tool activity, below). **The decision:** - `outcome` is `selected` or `cancelled`. @@ -178,6 +197,16 @@ it does not prove the process lost access. That happens at `run.settled`. - `decisionCardId` is set when the decision came through a decision card. - `grantId` ties the decision to its `grant.issued`. +**`decision.ruled`** records a ruling on a decision card that is not tied to +a parked tool call, such as whether to retry a stuck attempt. +- `decisionCardId` is required. +- `value` is the ruling: a string, number or boolean. +- `decidedBy` is required. It is a user id, or `"policy"`. +- `decidedByRole` is required, and comes from the same enum. + +A ruling on a card that a `permission.requested` carried belongs with that +request, and the OTLP export places it there. + ### Attempt | kind | data | default actor | @@ -200,7 +229,7 @@ it does not prove the process lost access. That happens at `run.settled`. | kind | data | default actor | |---|---|---| -| `tool.call` | `{toolCallId, title, kind?, status?, name?, surface?}` | run | +| `tool.call` | `{toolCallId, title, kind?, status?, name?, surface?, argsDigest?}` | run | | `tool.update` | `{toolCallId, status?, title?}` | run | - `kind` is the ACP tool kind (`read`, `edit`, `execute`, `fetch`, …). @@ -211,6 +240,21 @@ it does not prove the process lost access. That happens at `run.settled`. - `native`: the CLI's built-in tools. All three land in the same record. +- `argsDigest` is a digest of the call's arguments, never the arguments: + `"sha256:"` followed by the lowercase hex SHA-256 of the arguments' + RFC 8785 canonical form, the same serialization as a line. Key order does + not change it. For example, `{"b":2,"a":1}` is digested as the bytes + `{"a":1,"b":2}`. + - A writer refuses any other value, so raw arguments can't ride in under + this name. + - The same arguments give the same digest on `tool.call`, + `permission.requested` and `permission.decided`, so a reader can match a + call to the approval it ran under. + - The valet writes it on both permission entries when the ACP tool call + carries `rawInput`. An adapter writes it on `tool.call` with + `argsDigestOf(rawInput)`. + - A digest of low-entropy arguments can be confirmed by guessing them. It + hides the values from a casual reader, not from someone who can guess them. ### Credentials @@ -255,7 +299,8 @@ store writes the record kind: What's new or changed in `EVENT_TYPES`: - `task.claimed`, `task.released`, `task.done`, `grant.issued`, - `grant.revoked`, `attempt.settled` and `result.reported` are new entries. + `grant.revoked`, `attempt.settled`, `decision.ruled` and + `result.reported` are new entries. - `emit` also accepts the record kind `attempt.started`. - `readRun` returns each entry with its trace name as `type` (`run.started` for an `attempt.started` line), and puts the envelope fields beside it. @@ -282,6 +327,15 @@ What's new or changed in `EVENT_TYPES`: Adding a kind, an optional field or an enum value doesn't break anything, and keeps `v: 1`. A reader refuses a `v` higher than it knows. + + The 2026-10-25 revision kept `v: 1`. It added `decision.ruled`, + `argsDigest`, and `grantId` and `decisionCardId` on + `permission.requested`. It also made `question` and `askedOf` required + for writers of `permission.requested`. + - Readers must not assume those two fields on lines written before the + revision. + - The verifier checks the envelope and the chain, never data, so those + lines still verify. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -329,7 +383,9 @@ operates it. - **A tool call is a child span of its attempt:** `execute_tool {tool name}`. - **Decisions are span events.** A `permission.*` entry goes on the tool span with the same `toolCallId`. Without one, it goes on the current attempt, - then the run. + then the run. A `decision.ruled` entry goes on the span of the request + that carried its `decisionCardId`, and otherwise on the span current at + the time. - **Every other kind is an event on the span current at the time.** That covers grants, credentials, claims, `task.done`, results, liveness and `tool.update`. @@ -386,13 +442,14 @@ to that commit. `error.type` is Stable. - these named attributes: `seats.run.id`, `seats.task.id`, `seats.pod.id`, `seats.run.outcome`, `seats.run.attempts`, `seats.attempt.number`, `seats.attempt.outcome`, `seats.adapter`, `seats.cli.name`, - `seats.cli.version`, `seats.agent.owner`, `seats.tool.kind` and - `seats.tool.surface`; + `seats.cli.version`, `seats.agent.owner`, `seats.tool.kind`, + `seats.tool.surface` and `seats.tool.args_digest`; - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner` and `.name`; - the event's data, flattened as `seats.<path>` with camelCase segments in snake_case. For example, `decidedBy` becomes `seats.decided_by`, and - `scope.connectionId` becomes `seats.scope.connection_id`. + `scope.connectionId` becomes `seats.scope.connection_id`, and + `askedOf.id` becomes `seats.asked_of.id`. ## 6. Not in v1 (open for v2) @@ -420,10 +477,14 @@ files drift from what the code writes. - `run-ana-0001.jsonl`: Ana's agent (opencode 1.19.0) claims `TASK-101` in `pod-checkout`. - Its grant attenuates Ben's Sentry connection grant. - - The first attempt is stuck, and the agent is retried. + - The first attempt is stuck. Ana rules `retry` on card `card-2b9` as + `agent-owner` (`decision.ruled`), and the agent is retried. - A `git push` is denied by the policy. - - A Sentry write asks for approval, and Ben approves it as - `credential-owner` through a decision card. + - The headline case: Ana's agent asks a question about a Sentry write. The + `permission.requested` line is tied to Ana's grant and to card + `card-7f3`, and is asked of Ben as `credential-owner`. Ben approves it on + that card. `tool.call` and both permission entries carry the same + `argsDigest`. - The run reports a PR. - `run-ben-0001.jsonl`: Ben's agent (goose 1.54.0) reviews the PR under a read grant bound to the task. The board then marks the task done, which diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index 9592997438..fcfd9a0824 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -11,6 +11,6 @@ export { validateGrant, matchToolCall, TOOL_KINDS, CREDENTIAL_SOURCES } from './ export { prepareRunEnv, BASE_ENV_ALLOWLIST } from './valet/env.js'; export { createPermissionPolicy, DEFAULT_APPROVE_TIMEOUT_MS } from './valet/permissions.js'; export { - RECORD_VERSION, RECORD_SCHEMA, KINDS, canonicalLine, appendEntry, verifyRecord, readRecord, hashLine, toolSurfaceOf, + RECORD_VERSION, RECORD_SCHEMA, KINDS, canonicalLine, appendEntry, verifyRecord, readRecord, hashLine, toolSurfaceOf, argsDigestOf, } from './record/format.js'; export { toOtlpJson } from './record/otel.js'; diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 406422e974..b01cde16f3 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -39,6 +39,7 @@ export const KINDS = Object.freeze({ 'grant.revoked': Object.freeze({ group: 'grant', actor: 'system' }), 'permission.requested': Object.freeze({ group: 'approval', actor: 'run' }), 'permission.decided': Object.freeze({ group: 'approval', actor: 'decider' }), + 'decision.ruled': Object.freeze({ group: 'approval', actor: 'decider' }), 'attempt.started': Object.freeze({ group: 'attempt', actor: 'run', trace: 'run.started' }), 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), @@ -109,6 +110,20 @@ export const canonicalLine = (entry) => { /** Lowercase hex SHA-256 over a line's UTF-8 bytes: what the next `prev` holds. */ export const hashLine = (line) => createHash('sha256').update(String(line), 'utf8').digest('hex'); +/** + * A tool call's arguments as a digest, never the values: `sha256:` + the + * lowercase hex SHA-256 of the arguments' RFC 8785 form. The same arguments + * give the same digest wherever they are seen, so a reader can match a + * tool.call to the permission entries that approved it. Returns undefined + * when there are no arguments, or they are not JSON (a function, NaN). + */ +const ARGS_DIGEST = /^sha256:[0-9a-f]{64}$/; +export const argsDigestOf = (args) => { + if (args === undefined) return undefined; + try { return `sha256:${hashLine(canonical(args))}`; } catch { return undefined; } +}; +export const isArgsDigest = (value) => typeof value === 'string' && ARGS_DIGEST.test(value); + /** null when the actor is well formed, else the reason. */ const actorProblem = (actor) => { if (actor === null) return null; @@ -144,6 +159,13 @@ const optionalText = (value, where) => { const userIdOrPolicy = (value, where) => { if (value !== undefined && value !== null && value !== 'policy' && !isUserId(value)) fail(`${where} must be a user id or "policy"`); }; +// Only the digest form: raw arguments would carry values. +const optionalArgsDigest = (data, kind) => { + if (data.argsDigest !== undefined && !isArgsDigest(data.argsDigest)) fail(`${kind}.argsDigest must be sha256:<64 hex>, a digest of the arguments, never the arguments`); +}; +const checkRole = (role, where) => { + if (!DECIDED_BY_ROLES.includes(role)) fail(`${where} must be one of ${DECIDED_BY_ROLES.join(', ')}`); +}; /** A grant's scope, in the shape of a room grant: exact tool names and a connection id. */ const checkScope = (scope) => { @@ -220,7 +242,29 @@ const CHECKS = { userIdOrPolicy(data.doneBy, 'task.done.doneBy'); return data; }, + 'permission.requested': (data) => { + if (!isText(data.question)) fail('permission.requested needs a question: the human-readable ask'); + if (!isPlainObject(data.askedOf)) fail('permission.requested needs askedOf: {id, role}'); + refuseUnknownKeys(data.askedOf, ['id', 'role'], 'permission.requested.askedOf.'); + if (!isUserId(data.askedOf.id)) fail('permission.requested.askedOf.id must be an id token, never a display name'); + checkRole(data.askedOf.role, 'permission.requested.askedOf.role'); + optionalText(data.grantId, 'permission.requested.grantId'); + optionalText(data.decisionCardId, 'permission.requested.decisionCardId'); + optionalArgsDigest(data, 'permission.requested'); + return { ...data, askedOf: { id: data.askedOf.id, role: data.askedOf.role } }; + }, + 'decision.ruled': (data) => { + if (!isText(data.decisionCardId)) fail('decision.ruled needs a decisionCardId'); + const { value } = data; + if (!(typeof value === 'string' || typeof value === 'boolean' || (typeof value === 'number' && Number.isFinite(value)))) { + fail('decision.ruled.value must be a string, number or boolean'); + } + if (data.decidedBy !== 'policy' && !isUserId(data.decidedBy)) fail('decision.ruled.decidedBy must be a user id or "policy"'); + checkRole(data.decidedByRole, 'decision.ruled.decidedByRole'); + return data; + }, 'permission.decided': (data) => { + optionalArgsDigest(data, 'permission.decided'); // A rule-decided call never had a person in the loop; a writer says so // rather than leave a reader to infer it from rule.list. const out = { ...data }; @@ -239,6 +283,7 @@ const CHECKS = { }, 'tool.call': (data) => { if (data.surface !== undefined && !TOOL_SURFACES.includes(data.surface)) fail(`tool.call.surface must be one of ${TOOL_SURFACES.join(', ')}`); + optionalArgsDigest(data, 'tool.call'); return data; }, 'credential.granted': (data) => ({ ...data, names: envNames(data.names ?? [], 'credential.granted.names') }), @@ -275,7 +320,8 @@ export const normalizeData = (kind, data = {}, envelope = {}) => { }; /** A grant's rules acting on their own: named by the grant when its id is a token. */ -const policyActor = (grantId) => ({ type: 'policy', id: isUserId(`grant:${grantId}`) ? `grant:${grantId}` : 'policy' }); +export const policyActorId = (grantId) => (isText(grantId) && isUserId(`grant:${grantId}`) ? `grant:${grantId}` : 'policy'); +const policyActor = (grantId) => ({ type: 'policy', id: policyActorId(grantId) }); /** The actor a writer records when the emitter names none. */ export const defaultActor = (kind, data = {}, runActor = null) => { diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js index a65acf1a77..5d415f834d 100644 --- a/packages/seats/src/record/otel.js +++ b/packages/seats/src/record/otel.js @@ -106,6 +106,7 @@ const spansOfRun = (entries) => { }); const spans = [run]; let attempt = null; // { span, number, tools: Map<toolCallId, span>, usage } + const cards = new Map(); // decisionCardId → the span its request sits on const end = (s, ts) => { if (s && s.endTimeUnixNano === null) s.endTimeUnixNano = nanos(ts); }; const closeAttempt = (ts) => { if (!attempt) return; @@ -167,7 +168,7 @@ const spansOfRun = (entries) => { ['gen_ai.operation.name', 'execute_tool'], ['gen_ai.tool.name', toolName], ['gen_ai.tool.call.id', data.toolCallId], ['gen_ai.agent.name', attempt?.agentName], - ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], + ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], ['seats.tool.args_digest', data.argsDigest], ]), }); spans.push(s); @@ -181,7 +182,12 @@ const spansOfRun = (entries) => { attempt.usage = data; attempt.span.events.push(eventOf(entry)); } else if (kind === 'permission.requested' || kind === 'permission.decided') { - (attempt?.tools.get(data.toolCallId) || current()).events.push(eventOf(entry)); + const target = attempt?.tools.get(data.toolCallId) || current(); + target.events.push(eventOf(entry)); + if (data.decisionCardId && !cards.has(data.decisionCardId)) cards.set(data.decisionCardId, target); + } else if (kind === 'decision.ruled') { + // A ruling on a card that a request rode on joins that request; any other is an event where it happened. + (cards.get(data.decisionCardId) || current()).events.push(eventOf(entry)); } else { current().events.push(eventOf(entry)); } diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index 6f6d0a8fb3..2069eac6d0 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -14,10 +14,10 @@ export const EVENT_TYPES = Object.freeze([ 'turn.prompt_sent', // { chars } 'agent.message_chunk', // { text } 'agent.thought_chunk', // { chars } thoughts are counted, not stored - 'tool.call', // { toolCallId, title, kind?, status? } + 'tool.call', // { toolCallId, title, kind?, status?, name?, surface?, argsDigest? } 'tool.update', // { toolCallId, status?, title? } 'plan', // { entries: [{content, status}] } - 'permission.requested', // { toolCallId?, options: [{optionId, kind}] } + 'permission.requested', // { toolCallId?, options: [{optionId, kind}], question, askedOf: {id, role}, grantId?, decisionCardId?, argsDigest? } 'permission.decided', // { outcome: 'selected'|'cancelled', optionId?, kind?, toolCallId?, rule?: {list, pattern}, approval? } 'usage', // { inputTokens?, outputTokens?, totalTokens?, cost? } null-not-zero when unknown 'liveness.stalled', // { idleMs, windowMs } @@ -32,6 +32,7 @@ export const EVENT_TYPES = Object.freeze([ 'grant.issued', // { grantId, parentGrantId?, scope: {connectionId?, tools: [names], writeMode?, budget?, audience}, target?, issuedBy?, expiresAt?, binding? } 'grant.revoked', // { grantIds: [...], reason: 'task.done'|'expired'|'cascade'|'manual' } 'attempt.settled', // { attempt, outcome, failure?: {kind, detail} } + 'decision.ruled', // { decisionCardId, value, decidedBy, decidedByRole } a card ruling not tied to a parked tool call 'result.reported', // { status: 'done'|'partial'|'blocked'|'failed', refs?: [{type, id}] } ]); diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js index 260a18372d..21a08992d1 100644 --- a/packages/seats/src/valet/permissions.js +++ b/packages/seats/src/valet/permissions.js @@ -15,7 +15,7 @@ * An allow rule with no allow_once option therefore denies. */ import { traceOf } from '../trace/events.js'; -import { DECIDED_BY_ROLES, isUserId } from '../record/format.js'; +import { argsDigestOf, DECIDED_BY_ROLES, isUserId, policyActorId } from '../record/format.js'; import { matchToolCall, validateGrant } from './grant.js'; export const DEFAULT_APPROVE_TIMEOUT_MS = 5 * 60 * 1000; @@ -94,27 +94,75 @@ const decider = (approval) => { return { decidedBy: 'policy', decidedByRole: 'policy' }; }; +const isAsker = (value) => isPlainObject(value) && isUserId(value.id) && DECIDED_BY_ROLES.includes(value.role); + /** - * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal?, grantId? }) + * Who an ask goes to, for permission.requested. `askedOf` is {id, role} or + * ({toolCall, grantRule}) => {id, role, decisionCardId?}. A call a + * rule decides is asked of the grant's policy. A function that throws or + * answers malformed is recorded as asked of the policy too: the record does + * not name a person it cannot vouch for. + */ +const askOf = (askedOf, input, policy) => { + if (input.grantRule?.list !== 'ask') return policy; + let answer = askedOf; + if (typeof askedOf === 'function') { + try { answer = askedOf(input); } catch { answer = null; } + } + if (!isAsker(answer)) return policy; + return { + id: answer.id, + role: answer.role, + ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), + }; +}; + +/** + * createPermissionPolicy(grant, { approve?, trace?, approveTimeoutMs?, signal?, grantId?, askedOf? }) * → (request) => Promise<{outcome:'selected', optionId} | {outcome:'cancelled'}> * * `signal` is the run's AbortSignal: a cancelled run answers a pending ask * with cancelled, which is what ACP asks of a client after session/cancel. * `grantId` is recorded on each decision, to tie it to its grant.issued. + * With `askedOf` (above), each request is recorded first as + * permission.requested: the question (the tool call's title), who it is + * asked of, and the card it rides on. The approver receives that card id. + * When the tool call carries `rawInput`, both entries carry its argsDigest. */ export const createPermissionPolicy = (grant, { - approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, grantId, + approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, grantId, askedOf, } = {}) => { const valid = validateGrant(grant); const hook = traceOf({ trace }); + if (askedOf !== undefined && typeof askedOf !== 'function' && !isAsker(askedOf)) { + throw new Error('askedOf must be {id: <user id>, role: <decidedByRole>} or a function returning one'); + } + const policy = { id: policyActorId(grantId), role: 'policy' }; return async (request = {}) => { const toolCall = request?.toolCall || {}; const options = request?.options; const { decision, rule } = matchToolCall(valid, toolCall); + const toolCallId = typeof toolCall.toolCallId === 'string' ? toolCall.toolCallId : null; + const argsDigest = argsDigestOf(toolCall.rawInput); + let asker = null; + if (askedOf !== undefined) { + asker = askOf(askedOf, { toolCall, grantRule: rule }, policy); + const fallback = `${isText(toolCall.kind) ? toolCall.kind : 'tool'} call`; + hook.emit('permission.requested', { + toolCallId, + options: (Array.isArray(options) ? options : []).filter(isPlainObject).map(({ optionId, kind }) => ({ optionId, kind })), + question: isText(toolCall.title) ? toolCall.title : fallback, + askedOf: { id: asker.id, role: asker.role }, + ...(grantId ? { grantId } : {}), + ...(asker.decisionCardId ? { decisionCardId: asker.decisionCardId } : {}), + ...(argsDigest ? { argsDigest } : {}), + }); + } let approval; let answer; if (decision === 'ask') { - approval = await awaitApproval(approve, { toolCall, grantRule: rule }, { timeoutMs: approveTimeoutMs, signal }); + const input = { toolCall, grantRule: rule, ...(asker?.decisionCardId ? { decisionCardId: asker.decisionCardId } : {}) }; + approval = await awaitApproval(approve, input, { timeoutMs: approveTimeoutMs, signal }); if (approval.verdict === 'aborted') answer = { outcome: 'cancelled' }; else answer = approval.verdict === 'approved' ? allowAnswer(options) : null; } else if (decision === 'allow') { @@ -124,10 +172,11 @@ export const createPermissionPolicy = (grant, { hook.emit('permission.decided', { outcome: answer.outcome, ...(answer.optionId ? { optionId: answer.optionId, kind: answer.kind } : {}), - toolCallId: typeof toolCall.toolCallId === 'string' ? toolCall.toolCallId : null, + toolCallId, rule, ...(approval ? { approval: approval.verdict, ...decider(approval) } : {}), ...(grantId ? { grantId } : {}), + ...(argsDigest ? { argsDigest } : {}), }); return answer.outcome === 'selected' ? { outcome: 'selected', optionId: answer.optionId } : { outcome: 'cancelled' }; }; diff --git a/packages/seats/test-support/record-sample.js b/packages/seats/test-support/record-sample.js index cc962c82f1..ba3bc3102b 100644 --- a/packages/seats/test-support/record-sample.js +++ b/packages/seats/test-support/record-sample.js @@ -2,9 +2,9 @@ * Builds docs/examples: one task, two people, two agents on different CLIs. * * Ana's opencode agent fixes TASK-101 under a grant that attenuates Ben's - * Sentry connection grant. Its first attempt hangs and is retried; a - * `git push` is denied by the grant; a Sentry write asks, and Ben approves - * it as the credential's owner through a decision card. Ben's goose agent + * Sentry connection grant. Its first attempt hangs, and Ana rules on a card + * to retry it; a `git push` is denied by the grant; a Sentry write is asked + * of Ben on a decision card, and he approves it as the credential's owner. Ben's goose agent * then reviews the PR, and the board marks the task done, which ends the * review grant bound to it. * @@ -17,6 +17,7 @@ import { createTraceStore } from '../src/trace/store.js'; import { prepareRunEnv } from '../src/valet/env.js'; import { createPermissionPolicy } from '../src/valet/permissions.js'; import { toOtlpJson } from '../src/record/otel.js'; +import { argsDigestOf } from '../src/record/format.js'; export const SAMPLE_SECRETS = Object.freeze(['sk-or-sample-0000aaaa1111bbbb', 'sntry-proxy-sample-2222cccc3333', 'sk-sample-openai-4444dddd5555']); export const SAMPLE_RUN_IDS = Object.freeze(['run-ana-0001', 'run-ben-0001']); @@ -27,13 +28,17 @@ const OPTIONS = [ { optionId: 'always', name: 'Always allow', kind: 'allow_always' }, { optionId: 'reject', name: 'Reject', kind: 'reject_once' }, ]; -const options = OPTIONS.map(({ optionId, kind }) => ({ optionId, kind })); -/** A tool call the way an ACP adapter would record it, decided by the run's policy. */ -const toolCall = async (run, decide, call, finalStatus) => { - run.emit('tool.call', { ...call, status: 'pending' }); - run.emit('permission.requested', { toolCallId: call.toolCallId, options }); - const answer = await decide({ sessionId: 'ses', toolCall: { toolCallId: call.toolCallId, title: call.title, kind: call.kind }, options: OPTIONS }); +/** + * A tool call the way an ACP adapter would record it (with the digest of its + * arguments, never the arguments), then decided by the run's policy, which + * records the request and the decision. + */ +const toolCall = async (run, decide, { rawInput, ...call }, finalStatus) => { + run.emit('tool.call', { ...call, status: 'pending', argsDigest: argsDigestOf(rawInput) }); + const answer = await decide({ + sessionId: 'ses', toolCall: { toolCallId: call.toolCallId, title: call.title, kind: call.kind, rawInput }, options: OPTIONS, + }); const allowed = answer.outcome === 'selected' && answer.optionId === 'allow'; run.emit('tool.update', { toolCallId: call.toolCallId, status: allowed ? finalStatus : 'failed' }); }; @@ -77,7 +82,9 @@ export const buildSample = async (root) => { const anaDecide = createPermissionPolicy(anaGrant, { trace: ana, grantId: 'grant-ana-sentry-1', - approve: async () => ({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-7f3' }), + // An ask under this grant goes to Ben, who owns the Sentry connection, on a card. + askedOf: () => ({ id: 'u-ben', role: 'credential-owner', decisionCardId: 'card-7f3' }), + approve: async ({ decisionCardId }) => ({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId }), }); ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, attempt: 1 }); @@ -87,15 +94,21 @@ export const buildSample = async (root) => { ana.emit('liveness.stalled', { idleMs: 90000, windowMs: 90000 }); ana.emit('liveness.recovery', { action: 'cancel', attempt: 1 }); ana.emit('attempt.settled', { attempt: 1, outcome: 'stuck', failure: { kind: 'stuck', detail: 'no progress signal for 90000ms (window 90000ms)' } }); + ana.emit('decision.ruled', { decisionCardId: 'card-2b9', value: 'retry', decidedBy: 'u-ana', decidedByRole: 'agent-owner' }); ana.emit('liveness.recovery', { action: 'retry', attempt: 2 }); ana.emit('run.started', { adapter: 'opencode', cli: { name: 'opencode', version: '1.19.0' }, sessionId: 'ses-ana-2', attempt: 2 }); - await toolCall(ana, anaDecide, { toolCallId: 'tc-2', title: 'git push origin fix/total-rounding', kind: 'execute', surface: 'native' }, 'completed'); await toolCall(ana, anaDecide, { - toolCallId: 'tc-3', title: 'sentry.resolve_issue CHECKOUT-88', kind: 'fetch', name: 'sentry.resolve_issue', surface: 'connector', + toolCallId: 'tc-2', title: 'git push origin fix/total-rounding', kind: 'execute', surface: 'native', + rawInput: { command: 'git push origin fix/total-rounding' }, + }, 'completed'); + await toolCall(ana, anaDecide, { + toolCallId: 'tc-3', title: 'Resolve Sentry issue CHECKOUT-88: the half-cent rounding is fixed in PR #212', kind: 'fetch', + name: 'sentry.resolve_issue', surface: 'connector', rawInput: { issueId: 'CHECKOUT-88', status: 'resolved' }, }, 'completed'); await toolCall(ana, anaDecide, { - toolCallId: 'tc-4', title: 'commonly_post_message', kind: 'other', name: 'commonly_post_message', surface: 'commonly', + toolCallId: 'tc-4', title: 'commonly_post_message: the fix summary, to the pod', kind: 'other', name: 'commonly_post_message', surface: 'commonly', + rawInput: { podId: 'pod-checkout', content: 'Fixed in #212' }, }, 'completed'); ana.emit('agent.message_chunk', { text: 'Fixed the half-cent rounding in total.js and resolved CHECKOUT-88. PR #212 is open.' }); ana.emit('usage', { inputTokens: 18234, outputTokens: 2210 }); @@ -125,10 +138,11 @@ export const buildSample = async (root) => { }, }); await benEnv.cleanup(); - const benDecide = createPermissionPolicy(benGrant, { trace: ben, grantId: 'grant-ben-review-1' }); + const benDecide = createPermissionPolicy(benGrant, { trace: ben, grantId: 'grant-ben-review-1', askedOf: { id: 'u-ben', role: 'agent-owner' } }); ben.emit('run.started', { adapter: 'goose', cli: { name: 'goose', version: '1.54.0' }, attempt: 1 }); await toolCall(ben, benDecide, { - toolCallId: 'tc-1', title: 'github.get_pull_request #212', kind: 'read', name: 'github.get_pull_request', surface: 'connector', + toolCallId: 'tc-1', title: 'Read pull request #212', kind: 'read', name: 'github.get_pull_request', surface: 'connector', + rawInput: { pullNumber: 212 }, }, 'completed'); ben.emit('agent.message_chunk', { text: 'Reviewed #212: the rounding fix is right and the half-cent case is tested.' }); ben.emit('attempt.settled', { attempt: 1, outcome: 'ok' }); diff --git a/packages/seats/test/record-example.test.js b/packages/seats/test/record-example.test.js index 6bb16a991f..931b25c91f 100644 --- a/packages/seats/test/record-example.test.js +++ b/packages/seats/test/record-example.test.js @@ -60,6 +60,19 @@ test('the headline case: Ana\'s agent asks, Ben approves as the credential owner assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }); const issued = entries.find((e) => e.kind === 'grant.issued'); assert.equal(asked.data.grantId, issued.data.grantId); + // The ask itself: Ana's agent asks Ben, tied to the grant and the card, about this exact call. + const request = entries.find((e) => e.kind === 'permission.requested' && e.data.toolCallId === asked.data.toolCallId); + assert.equal(request.actor.owner, 'u-ana'); + assert.deepEqual(request.data.askedOf, { id: 'u-ben', role: 'credential-owner' }); + assert.equal(request.data.grantId, issued.data.grantId); + assert.equal(request.data.decisionCardId, 'card-7f3'); + assert.match(request.data.question, /CHECKOUT-88/); + const call = entries.find((e) => e.kind === 'tool.call' && e.data.toolCallId === asked.data.toolCallId); + assert.match(call.data.argsDigest, /^sha256:[0-9a-f]{64}$/); + assert.equal(request.data.argsDigest, call.data.argsDigest); + assert.equal(asked.data.argsDigest, call.data.argsDigest); + const ruled = entries.find((e) => e.kind === 'decision.ruled'); + assert.deepEqual(ruled.data, { decidedBy: 'u-ana', decidedByRole: 'agent-owner', decisionCardId: 'card-2b9', value: 'retry' }); assert.equal(issued.data.scope.connectionId, 'conn-ben-sentry'); assert.equal(issued.data.parentGrantId, 'grant-ben-sentry-root'); const denied = entries.find((e) => e.kind === 'permission.decided' && e.data.rule.list === 'deny'); diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 7cd6b10777..65067fe3a4 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { - appendEntry, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -252,3 +252,58 @@ test('the writer refuses unknown kinds, a bad seq, and a non-UTC ts', () => { assert.throws(() => appendEntry(null, { ...ENV, ts: ts(), kind: 'usage', data: {}, seq: 2 }), /seq 2/); assert.throws(() => appendEntry(null, { ...ENV, ts: '2026-10-25T12:00:00+02:00', kind: 'usage', data: {} }), /RFC 3339/); }); + +const request = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.requested', data }, { runActor: AGENT })); +const askData = (overrides = {}) => ({ + toolCallId: 'tc-1', options: [{ optionId: 'allow', kind: 'allow_once' }], question: 'Resolve CHECKOUT-88?', + askedOf: { id: 'u-ben', role: 'credential-owner' }, grantId: 'g-1', decisionCardId: 'card-1', ...overrides, +}); + +test('permission.requested: question and askedOf are required, askedOf.id is an id and its role is from the enum', () => { + const entry = request(askData()); + assert.deepEqual(entry.data.askedOf, { id: 'u-ben', role: 'credential-owner' }); + assert.equal(entry.data.question, 'Resolve CHECKOUT-88?'); + assert.deepEqual(entry.actor, AGENT, 'the agent asks'); + const { askedOf, question, ...rest } = askData(); + assert.throws(() => request(rest), /question/); + assert.throws(() => request({ ...rest, question }), /askedOf/); + assert.throws(() => request(askData({ question: '' })), /question/); + assert.throws(() => request(askData({ askedOf: { id: 'Ben Smith', role: 'credential-owner' } })), /askedOf.id/); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben' } })), /askedOf.role/); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'boss' } })), /askedOf.role/); + assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'policy', email: 'ben@example.com' } })), /unknown field/); + assert.throws(() => request(askData({ decisionCardId: '' })), /decisionCardId/); + assert.equal(request(askData({ askedOf: { id: 'grant:g-1', role: 'policy' } })).data.askedOf.id, 'grant:g-1'); +}); + +test('decision.ruled: a card ruling, validated like permission.decided', () => { + const rule = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'decision.ruled', data })); + const good = { decisionCardId: 'card-2', value: 'retry', decidedBy: 'u-ana', decidedByRole: 'agent-owner' }; + const entry = rule(good); + assert.deepEqual(entry.data, good); + assert.deepEqual(entry.actor, { type: 'human', id: 'u-ana' }, 'the ruler is the actor'); + assert.deepEqual(rule({ ...good, decidedBy: 'policy', decidedByRole: 'policy' }).actor, { type: 'policy', id: 'policy' }); + for (const value of [true, 3, 'ship']) rule({ ...good, value }); + assert.throws(() => rule({ ...good, decisionCardId: undefined }), /decisionCardId/); + assert.throws(() => rule({ ...good, decidedBy: 'Ana Smith' }), /decidedBy/); + assert.throws(() => rule({ ...good, decidedBy: undefined }), /decidedBy/); + assert.throws(() => rule({ ...good, decidedByRole: 'owner' }), /decidedByRole/); + assert.throws(() => rule({ ...good, decidedByRole: null }), /decidedByRole/); + for (const value of [undefined, null, { a: 1 }, ['x'], Number.NaN]) assert.throws(() => rule({ ...good, value }), /value/, `value ${value}`); +}); + +test('argsDigest is sha256 of the canonical args, and anything else is refused, so raw args cannot ride in', () => { + const digest = argsDigestOf({ b: 2, a: [1, 'x'] }); + assert.equal(digest, `sha256:${createHash('sha256').update('{"a":[1,"x"],"b":2}').digest('hex')}`); + assert.equal(argsDigestOf({ a: [1, 'x'], b: 2 }), digest, 'key order does not change it'); + assert.equal(argsDigestOf(undefined), undefined); + assert.equal(argsDigestOf({ n: Number.NaN }), undefined); + const call = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', argsDigest } }); + const decided = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data: { outcome: 'cancelled', toolCallId: 't', argsDigest } }); + for (const write of [call, decided, (d) => request(askData({ argsDigest: d }))]) { + write(digest); + for (const raw of [{ command: 'git push' }, 'git push origin main', `sha256:${'A'.repeat(64)}`, `sha256:${'a'.repeat(63)}`, `sha512:${'a'.repeat(64)}`, 'a'.repeat(64), ['x'], 42]) { + assert.throws(() => write(raw), /argsDigest/, `raw ${JSON.stringify(raw)}`); + } + } +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index 0c52f346b4..25866be919 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -52,7 +52,7 @@ test('OTLP/JSON structure: ids, times, kinds, typed attributes', async () => { } } // Content stays out: no message text anywhere in the export. - assert.ok(!JSON.stringify(otlp).includes('half-cent rounding')); + assert.ok(!JSON.stringify(otlp).includes('Fixed the half-cent rounding in total.js')); }); test('one task is one trace; ids follow the documented derivation', async () => { @@ -129,3 +129,27 @@ test('a run without a task is its own trace; unknown kinds and newer versions ar assert.equal(attrOf(spans[0], 'error.type').stringValue, 'crashed'); assert.equal(attrOf(spans[1], 'error.type').stringValue, 'crashed', 'an unsettled last attempt takes the run\'s failure'); }); + +test('decision.ruled joins the tool span of the request on its card, else the span current at the time; argsDigest is a tool attribute', async () => { + const { entries } = await sample(); + const spans = spansOf(toOtlpJson(entries)); + const resolve = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.match(attrOf(resolve, 'seats.tool.args_digest').stringValue, /^sha256:[0-9a-f]{64}$/); + const asked = resolve.events.find((e) => e.name === 'permission.requested'); + assert.equal(attrOf(asked, 'seats.asked_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(asked, 'seats.decision_card_id').stringValue, 'card-7f3'); + assert.equal(attrOf(asked, 'seats.args_digest').stringValue, attrOf(resolve, 'seats.tool.args_digest').stringValue); + // The retry ruling rode no tool call: it lands on Ana's run span, between attempts. + const anaRun = spans.find((s) => s.name === 'seats.run' && attrOf(s, 'seats.run.id').stringValue === 'run-ana-0001'); + const ruled = anaRun.events.find((e) => e.name === 'decision.ruled'); + assert.equal(attrOf(ruled, 'seats.value').stringValue, 'retry'); + assert.equal(attrOf(ruled, 'seats.decided_by_role').stringValue, 'agent-owner'); + // A ruling on a card a request carried goes to that request's tool span. + const ana = entries.filter((e) => e.runId === 'run-ana-0001'); + const settle = ana.at(-1); + const extra = [...ana.slice(0, -1), { + ...settle, kind: 'decision.ruled', data: { decisionCardId: 'card-7f3', value: 'approved', decidedBy: 'u-ben', decidedByRole: 'credential-owner' }, + }, { ...settle, seq: settle.seq + 1 }]; + const again = spansOf(toOtlpJson(extra)).find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.ok(again.events.some((e) => e.name === 'decision.ruled')); +}); diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js index a9458e00e2..905c43dbeb 100644 --- a/packages/seats/test/record-valet.test.js +++ b/packages/seats/test/record-valet.test.js @@ -6,6 +6,7 @@ import { join } from 'node:path'; import { createTraceStore } from '../src/trace/store.js'; import { prepareRunEnv } from '../src/valet/env.js'; import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { argsDigestOf } from '../src/record/format.js'; const FROM_ENV = 'sk-env-planted-0a1b2c3d4e5f'; const FROM_FILE = 'sk-file-planted-6a7b8c9d0e1f'; @@ -143,3 +144,63 @@ test('no answer in time is decided by policy', async () => { assert.equal(events[0].data.decidedBy, 'policy'); assert.equal(events[0].data.decidedByRole, 'policy'); }); + +test('the question is redacted: a credential value in the ask never reaches disk', () => withRoot(async (root) => { + const store = createTraceStore({ root: join(root, 'trace') }); + const run = store.startRun({ runId: 'run-va-0003', actor: { type: 'agent', id: 'u-agent-1', owner: 'u-ana' } }); + const grant = { credentials: [{ name: 'BROKER_TOKEN', from: 'resolver' }], tools: { ask: ['fetch'] } }; + const env = await prepareRunEnv({ grant, resolveSecret: async () => FROM_RESOLVER, root, trace: run }); + await env.cleanup(); + const decide = createPermissionPolicy(grant, { + trace: run, grantId: 'g-3', askedOf: { id: 'u-ben', role: 'credential-owner' }, approve: async () => ({ approved: true, by: 'u-ben' }), + }); + await decide({ toolCall: { toolCallId: 'tc-1', title: `Call the API with token ${FROM_RESOLVER}?`, kind: 'fetch' }, options: OPTIONS }); + const raw = readFileSync(join(root, 'trace', 'runs', 'run-va-0003.jsonl'), 'utf8'); + const asked = store.readRun('run-va-0003').find((e) => e.kind === 'permission.requested'); + assert.equal(asked.data.question, 'Call the API with token [redacted]?', 'positive control: the question was written, scrubbed'); + assert.ok(!raw.includes(FROM_RESOLVER)); + assert.equal(store.verifyRun('run-va-0003').ok, true); +})); + +const asks = async (askedOf, toolCall = { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, tools = { ask: ['fetch'] }) => { + const events = []; + const decide = createPermissionPolicy({ tools }, { + trace: { emit: (type, data) => events.push({ type, data }) }, grantId: 'g-5', askedOf, + approve: async (input) => { events.push({ type: 'approver-saw', data: input }); return { approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: input.decisionCardId }; }, + }); + await decide({ toolCall, options: OPTIONS }); + return events; +}; + +test('permission.requested is written first, asked of the host\'s choice for an ask and of the policy for a rule', async () => { + const events = await asks(() => ({ id: 'u-ben', role: 'credential-owner', decisionCardId: 'card-9' })); + assert.deepEqual(events.map((e) => e.type), ['permission.requested', 'approver-saw', 'permission.decided']); + assert.deepEqual(events[0].data, { + toolCallId: 'tc-1', options: OPTIONS.map(({ optionId, kind }) => ({ optionId, kind })), question: 'fetch it', + askedOf: { id: 'u-ben', role: 'credential-owner' }, grantId: 'g-5', decisionCardId: 'card-9', + }); + assert.equal(events[1].data.decisionCardId, 'card-9', 'the approver gets the card the ask rides on'); + assert.equal(events[2].data.decisionCardId, 'card-9'); + const byRule = await asks({ id: 'u-ben', role: 'credential-owner' }, { toolCallId: 'tc-2', title: 'cat file', kind: 'read' }, { allow: ['read'] }); + assert.deepEqual(byRule[0].data.askedOf, { id: 'grant:g-5', role: 'policy' }); + assert.equal(byRule.some((e) => e.type === 'approver-saw'), false); + for (const bad of [() => ({ id: 'Ben Smith', role: 'credential-owner' }), () => ({ id: 'u-ben' }), () => { throw new Error('x'); }]) { + assert.deepEqual((await asks(bad))[0].data.askedOf, { id: 'grant:g-5', role: 'policy' }, 'a malformed answer names nobody'); + } + assert.throws(() => createPermissionPolicy({ tools: {} }, { askedOf: { id: 'Ben Smith', role: 'policy' } }), /askedOf/); + const untitled = await asks({ id: 'u-ben', role: 'agent-owner' }, { toolCallId: 'tc-3', kind: 'fetch' }); + assert.equal(untitled[0].data.question, 'fetch call'); +}); + +test('the permission path writes the digest of the call\'s arguments, never the arguments', async () => { + const rawInput = { url: 'https://example.test/issues/88', body: { status: 'resolved' } }; + const events = await asks({ id: 'u-ben', role: 'credential-owner' }, { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch', rawInput }); + const digest = argsDigestOf(rawInput); + assert.match(digest, /^sha256:[0-9a-f]{64}$/); + assert.equal(events[0].data.argsDigest, digest); + assert.equal(events[2].data.argsDigest, digest); + assert.ok(!JSON.stringify([events[0].data, events[2].data]).includes('example.test')); + const none = await asks(undefined, { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }); + assert.equal('argsDigest' in none[1].data, false, 'no rawInput, no digest'); + assert.deepEqual(none.map((e) => e.type), ['approver-saw', 'permission.decided'], 'without askedOf, nothing new is emitted'); +}); From 1515448f50075a2f9c593d571103b8ac79b483c9 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:15:11 -0700 Subject: [PATCH 06/20] feat(seats): record v1 actor carries onBehalfOf, the human an agent acts for ADR-025 D18 wants an action's audit trail to hold the pair (agent, on-behalf-of human). actor.owner says only whose agent it is, so add an optional actor.onBehalfOf: {id, role}, allowed on agent actors only. id follows the actor.id token rule; role is the closed set ON_BEHALF_ROLES = requester | assigner. A non-agent actor carrying it, a bad id, an unknown role or an extra key is a malformed actor, refused by the writer and reported as bad_envelope (actor) by the verifier. It lives on the actor, not in kind data, so tool.call and permission.requested carry it like every other entry the agent authors. The OTLP export maps it beside actor.owner: seats.actor.on_behalf_of.id and .role on events, seats.agent.on_behalf_of.id and .role on the attempt span. v stays 1; lines without the field are byte-identical, pinned by hash against the previous writer, and docs/examples did not change. The spec says a run with no human behind it writes no onBehalfOf and a writer never fills it with the owner. That is documented, not enforced: refusing id === owner would reject the real case where the owner is the requester. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 43 +++++++++++++++-- packages/seats/src/record/format.js | 18 ++++++- packages/seats/src/record/otel.js | 2 + packages/seats/test/record-format.test.js | 57 ++++++++++++++++++++++- packages/seats/test/record-otel.test.js | 26 +++++++++++ packages/seats/test/record-store.test.js | 13 ++++++ 6 files changed, 153 insertions(+), 6 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index b0b395f910..087703d8c3 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -42,7 +42,7 @@ Each line is one JSON object, followed by `\n` (UTF-8): | `data` | object | Kind-specific fields (section 2). | | `prev` | string or `null` | Lowercase hex SHA-256 of the previous line's bytes. `null` on `seq` 1. | -**`actor`** is `{type, id, owner?, name?}`: +**`actor`** is `{type, id, owner?, onBehalfOf?, name?}`: - `type` is `agent`, `human`, `policy` or `system`. - `id` is an identity, never a display name. - For `agent` and `human` it is the instance's user id. Agents are users too. @@ -54,6 +54,8 @@ Each line is one JSON object, followed by `\n` (UTF-8): can't pass for one. - `owner` is set on `agent` actors only. It is the user id of the person the agent belongs to. +- `onBehalfOf` is set on `agent` actors only, and is optional. It is the + human whose request the agent is acting on (below). - `name` is an optional display name. It is never part of the identity. The owner of an agent and the person who approves what it does are separate @@ -64,6 +66,35 @@ fields on purpose: In the case this format exists for, they differ: Ana's agent asks, and Ben approves. +**`actor.onBehalfOf`** is `{id, role}`. It records the pair an audit trail +needs for an agent's action: the agent, and the human it acted for +(ADR-025 D18). `owner` can't say this, because it only says whose agent it is. +- `id` is the human's user id, an id token under the same rule as `actor.id`. +- `role` is one of: + - `requester`: the human whose message started the run; + - `assigner`: the human who assigned the task. +- No other key is allowed. A writer throws on an unknown role or key, and a + verifier reports `bad_envelope` (`actor`) for a line that has one. +- It is allowed on `agent` actors only. Any other actor type carrying it is a + malformed actor. +- It lives on the actor, never in `data`, so it covers every entry the agent + authors (`tool.call` and `permission.requested` included), and the pair is + never split across two places. +- A run with no human behind it (a schedule, a heartbeat, an agent-to-agent + DM) writes no `onBehalfOf`. A writer never fills it in with the agent's + owner: `owner` already says whose agent it is, and `onBehalfOf` means a + human asked. The writer does not enforce this by comparing `id` to `owner`, + because the owner can really be the requester (Ana asks her own agent), and + that line must be accepted. + +Adding it kept `v: 1` (section 3, rule 3), and a line without it is +byte-identical to one written before the field existed. Ana's agent, acting +on a message from Ben, calls a Sentry tool: + +```json +{"actor":{"id":"u-agent-ana-opencode","onBehalfOf":{"id":"u-ben","role":"requester"},"owner":"u-ana","type":"agent"},"data":{"name":"sentry.resolve_issue","surface":"connector","title":"Resolve CHECKOUT-88","toolCallId":"tc-1"},"kind":"tool.call","podId":"pod-checkout","prev":"cbe33a93d273b1ad959fb6e836995ac8c9a119921ddbcce7b785c7d63ccf2a9e","runId":"run-ana-0002","schema":"seats.record","seq":2,"taskId":"TASK-101","ts":"2026-10-25T16:01:05.000Z","v":1} +``` + `actor: null` means the writer did not know who caused the entry. A reader must not fill it in. Each kind has a default actor (section 2), which a writer records when the emitter names none. @@ -336,6 +367,9 @@ What's new or changed in `EVENT_TYPES`: revision. - The verifier checks the envelope and the chain, never data, so those lines still verify. + + A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`. + Lines without it are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -442,10 +476,11 @@ to that commit. `error.type` is Stable. - these named attributes: `seats.run.id`, `seats.task.id`, `seats.pod.id`, `seats.run.outcome`, `seats.run.attempts`, `seats.attempt.number`, `seats.attempt.outcome`, `seats.adapter`, `seats.cli.name`, - `seats.cli.version`, `seats.agent.owner`, `seats.tool.kind`, + `seats.cli.version`, `seats.agent.owner`, `seats.agent.on_behalf_of.id`, + `seats.agent.on_behalf_of.role`, `seats.tool.kind`, `seats.tool.surface` and `seats.tool.args_digest`; - - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner` and - `.name`; + - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner`, + `.name`, `.on_behalf_of.id` and `.on_behalf_of.role`; - the event's data, flattened as `seats.<path>` with camelCase segments in snake_case. For example, `decidedBy` becomes `seats.decided_by`, and `scope.connectionId` becomes `seats.scope.connection_id`, and diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index b01cde16f3..0e5fa51588 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -21,6 +21,9 @@ export const WRITE_MODES = Object.freeze(['read', 'write', 'write-with-confirm'] export const GRANT_TARGET_KINDS = Object.freeze(['pod', 'seat']); export const OUTCOMES = Object.freeze(['ok', 'failed', 'stuck', 'cancelled']); export const RESULT_STATUSES = Object.freeze(['done', 'partial', 'blocked', 'failed']); +// Whose request an agent acts on: 'requester' sent the message that started +// the run, 'assigner' assigned its task. Absent when no human asked. +export const ON_BEHALF_ROLES = Object.freeze(['requester', 'assigner']); /** The actors a writer records for its own bookkeeping, and for the board. */ export const SYSTEM_ACTOR = Object.freeze({ type: 'system', id: 'seats' }); @@ -132,6 +135,15 @@ const actorProblem = (actor) => { if (!isUserId(actor.id)) return 'actor.id must be an id token (a user id for agent and human actors), never a display name'; if (actor.owner !== undefined && (actor.type !== 'agent' || !isUserId(actor.owner))) return 'actor.owner is a user id, on agent actors only'; if (actor.name !== undefined && !isText(actor.name)) return 'actor.name must be a non-empty string'; + if (actor.onBehalfOf !== undefined) { + const behalf = actor.onBehalfOf; + if (actor.type !== 'agent') return 'actor.onBehalfOf is on agent actors only'; + if (!isPlainObject(behalf)) return 'actor.onBehalfOf must be {id, role}'; + const extra = Object.keys(behalf).find((key) => key !== 'id' && key !== 'role'); + if (extra !== undefined) return `unknown field actor.onBehalfOf.${extra}`; + if (!isUserId(behalf.id)) return 'actor.onBehalfOf.id must be an id token (a user id), never a display name'; + if (!ON_BEHALF_ROLES.includes(behalf.role)) return `actor.onBehalfOf.role must be one of ${ON_BEHALF_ROLES.join(', ')}`; + } return null; }; @@ -140,7 +152,11 @@ export const validateActor = (actor) => { if (problem) fail(problem); if (actor === null) return null; return Object.freeze({ - type: actor.type, id: actor.id, ...(actor.owner ? { owner: actor.owner } : {}), ...(actor.name ? { name: actor.name } : {}), + type: actor.type, + id: actor.id, + ...(actor.owner ? { owner: actor.owner } : {}), + ...(actor.onBehalfOf ? { onBehalfOf: Object.freeze({ id: actor.onBehalfOf.id, role: actor.onBehalfOf.role }) } : {}), + ...(actor.name ? { name: actor.name } : {}), }); }; diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js index 5d415f834d..0d031d2a1a 100644 --- a/packages/seats/src/record/otel.js +++ b/packages/seats/src/record/otel.js @@ -71,6 +71,7 @@ const eventOf = (entry) => ({ ...(entry.actor ? attrs([ ['seats.actor.type', entry.actor.type], ['seats.actor.id', entry.actor.id], ['seats.actor.owner', entry.actor.owner], ['seats.actor.name', entry.actor.name], + ['seats.actor.on_behalf_of.id', entry.actor.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor.onBehalfOf?.role], ]) : []), ...flatten(entry.data), ], @@ -137,6 +138,7 @@ const spansOfRun = (entries) => { ['gen_ai.agent.id', entry.actor?.id], ['gen_ai.agent.name', agentName], ['gen_ai.conversation.id', data.sessionId], ['seats.agent.owner', entry.actor?.owner], + ['seats.agent.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.agent.on_behalf_of.role', entry.actor?.onBehalfOf?.role], ['seats.attempt.number', data.attempt], ['seats.adapter', data.adapter], ['seats.cli.name', data.cli?.name], ['seats.cli.version', data.cli?.version], ]), diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 65067fe3a4..47340612f0 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { - appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, ON_BEHALF_ROLES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -307,3 +307,58 @@ test('argsDigest is sha256 of the canonical args, and anything else is refused, } } }); + +const BEHALF = { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester' } }; + +test('actor.onBehalfOf: an agent entry carrying it hashes, chains and verifies, on every kind the agent authors', () => { + assert.deepEqual(ON_BEHALF_ROLES, ['requester', 'assigner']); + const lines = []; + const add = (kind, data) => lines.push(appendEntry(lines.at(-1) ?? null, { ...ENV, ts: ts(), kind, data }, { runActor: BEHALF })); + add('attempt.started', { attempt: 1, adapter: 'fake' }); + add('tool.call', { toolCallId: 'tc-1', title: 'resolve', surface: 'connector', argsDigest: argsDigestOf({ id: 88 }) }); + add('permission.requested', askData({ argsDigest: argsDigestOf({ id: 88 }) })); + lines.push(appendEntry(lines.at(-1), { + ...ENV, ts: ts(), kind: 'result.reported', data: { status: 'done' }, actor: { ...AGENT, onBehalfOf: { id: 'u-cy', role: 'assigner' } }, + })); + const result = verifyRecord(text(lines)); + assert.deepEqual(result.errors, []); + assert.equal(result.head, hashLine(lines.at(-1))); + const entries = readRecord(text(lines)); + for (const entry of entries.slice(0, 3)) assert.deepEqual(entry.actor, BEHALF, `${entry.kind} carries the pair on its actor`); + assert.deepEqual(entries[3].actor.onBehalfOf, { id: 'u-cy', role: 'assigner' }); + assert.equal(entries[2].data.onBehalfOf, undefined, 'the pair lives on the actor, never in data'); + assert.match(lines[1], /"actor":\{"id":"u-agent-1","name":"Ana's agent","onBehalfOf":\{"id":"u-ben","role":"requester"\},"owner":"u-ana","type":"agent"\}/); +}); + +test('actor.onBehalfOf: refused off agent actors, with a bad id, an unknown role or an extra key, by writer and verifier', () => { + const at = (actor) => appendEntry(null, { runId: RUN, ts: ts(), kind: 'usage', data: {}, actor }); + const bad = [ + [{ type: 'human', id: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } }, /agent actors only/], + [{ type: 'system', id: 'seats', onBehalfOf: { id: 'u-ben', role: 'requester' } }, /agent actors only/], + [{ ...AGENT, onBehalfOf: { id: 'Ben Smith', role: 'requester' } }, /onBehalfOf.id/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben' } }, /onBehalfOf.role/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben', role: 'owner' } }, /onBehalfOf.role/], + [{ ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester', email: 'ben@example.com' } }, /unknown field actor.onBehalfOf.email/], + [{ ...AGENT, onBehalfOf: 'u-ben' }, /onBehalfOf must be/], + [{ ...AGENT, onBehalfOf: null }, /onBehalfOf must be/], + ]; + const good = JSON.parse(at(BEHALF)); + for (const [actor, message] of bad) { + assert.throws(() => at(actor), message, JSON.stringify(actor)); + // A hand-written line with the same actor, canonical and chained, fails the envelope check. + const line = canonicalLine({ ...good, actor }); + assert.deepEqual(reasons(verifyRecord(`${line}\n`)), ['1:bad_envelope'], JSON.stringify(actor)); + assert.equal(verifyRecord(`${line}\n`).errors[0].detail, 'actor'); + } +}); + +test('actor.onBehalfOf is additive: a record written without it is byte-identical to the writer before the field', () => { + // Hashes pinned from the writer at b7b71f63a, before onBehalfOf existed. + const env = { runId: 'run-fmt-0001', taskId: 'TASK-7', podId: 'pod-a' }; + const l1 = appendEntry(null, { ...env, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', data: { attempt: 1, adapter: 'fake' } }, { runActor: AGENT }); + const l2 = appendEntry(l1, { ...env, ts: '2026-10-25T12:00:02.000Z', kind: 'tool.call', data: { toolCallId: 'tc-1', title: 'x', surface: 'native' } }, { runActor: AGENT }); + assert.equal(hashLine(l1), 'f91b90fc46781ca9b915b80cce5b9633f603e90742bfc8b1f6dc82d17e01c264'); + assert.equal(hashLine(l2), '1689f865735018eafddcb3fcf0b267d8ead7d9546d45b6d9a471c23eed21cfc6'); + assert.ok(!l1.includes('onBehalfOf') && !l2.includes('onBehalfOf')); + assert.deepEqual(verifyRecord(`${l1}\n${l2}\n`).errors, []); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index 25866be919..aa4cd6661e 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -153,3 +153,29 @@ test('decision.ruled joins the tool span of the request on its card, else the sp const again = spansOf(toOtlpJson(extra)).find((s) => s.name === 'execute_tool sentry.resolve_issue'); assert.ok(again.events.some((e) => e.name === 'decision.ruled')); }); + +test('actor.onBehalfOf maps beside actor.owner: on every event, and on the attempt span', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-obo-0001', prev: null }; + const actor = { type: 'agent', id: 'u-agent-ana', owner: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', actor, data: { attempt: 1 } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'tool.call', actor, data: { toolCallId: 'tc-1', name: 'sentry.resolve_issue' } }, + { + ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'permission.requested', actor, + data: { toolCallId: 'tc-1', question: 'Resolve?', askedOf: { id: 'u-ben', role: 'credential-owner' } }, + }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'run.settled', actor: { type: 'system', id: 'seats' }, data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const attempt = spans.find((s) => s.name.startsWith('invoke_agent')); + assert.equal(attrOf(attempt, 'seats.agent.owner').stringValue, 'u-ana'); + assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.role').stringValue, 'requester'); + const asked = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue').events.find((e) => e.name === 'permission.requested'); + assert.equal(attrOf(asked, 'seats.actor.owner').stringValue, 'u-ana'); + assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.id').stringValue, 'u-ben'); + assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.role').stringValue, 'requester'); + // Without the field, no attribute appears. + const plain = spansOf(toOtlpJson(entries.map((e) => (e.actor.type === 'agent' ? { ...e, actor: { type: 'agent', id: 'u-agent-ana', owner: 'u-ana' } } : e)))); + assert.ok(!JSON.stringify(plain).includes('on_behalf_of')); +}); diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js index d1dd6c478c..40ab9e7c55 100644 --- a/packages/seats/test/record-store.test.js +++ b/packages/seats/test/record-store.test.js @@ -149,3 +149,16 @@ test('redact: known values never reach a line, even inside agent text', () => wi assert.ok(raw.includes('[redacted]')); assert.equal(store.verifyRun('run-st-0012').ok, true, 'scrubbing happens before hashing'); })); + +test('the store writes actor.onBehalfOf through emit, and refuses it on a non-agent run actor', () => withStore((store, root) => { + const actor = { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'requester' } }; + const run = store.startRun({ runId: 'run-st-obo1', adapter: 'fake', taskId: 'TASK-9', podId: 'pod-a', actor }); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit('tool.call', { toolCallId: 'tc-1', title: 'resolve', surface: 'connector' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 5, attempts: 1 }); + const events = store.readRun('run-st-obo1'); + assert.deepEqual(events[1].actor, actor); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-obo1'), 'utf8')).errors, []); + assert.throws(() => store.startRun({ runId: 'run-st-obo2', actor: { type: 'human', id: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } } }), /agent actors only/); + assert.throws(() => store.startRun({ runId: 'run-st-obo3', actor: { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'boss' } } }), /onBehalfOf.role/); +})); From 41c3e99d8d80ed9681cc1faab1953859a4e2fd5d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:17:41 -0700 Subject: [PATCH 07/20] test(seats): args digest vectors for writers in other languages A broker written in another language that computes its own key-sorted digest drifts from argsDigestOf on undefined values and non-finite numbers. test/vectors/args-digest.json holds 31 vectors, {name, args, canonical, digest} or {name, args, error: true}: nested key order by UTF-16 code units, unicode and escaping, number formatting (1.0, -0, 1e20/1e21, 1e-7, extremes, past 2^53), arrays, and the divergent cases as argsDigestOf handles them today: undefined in an object drops the key, undefined in an array becomes null, NaN/Infinity give no digest. JSON can't hold undefined or NaN, so args use two tags, documented in _about: {"$undefined":true} and {"$number":"<literal>"}. Each canonical form was written by hand and checked against argsDigestOf when generating. test/args-digest-vectors.test.js decodes the tags and asserts every vector, so the file can't drift from the code. record-v1.md says a writer in another language must pass the file and omits argsDigest on arguments it can't digest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 12 + .../seats/test/args-digest-vectors.test.js | 59 ++++ packages/seats/test/vectors/args-digest.json | 318 ++++++++++++++++++ 3 files changed, 389 insertions(+) create mode 100644 packages/seats/test/args-digest-vectors.test.js create mode 100644 packages/seats/test/vectors/args-digest.json diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 087703d8c3..b1655d9e23 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -287,6 +287,18 @@ request, and the OTLP export places it there. - A digest of low-entropy arguments can be confirmed by guessing them. It hides the values from a casual reader, not from someone who can guess them. +**Writers in other languages.** A writer that computes `argsDigest` outside +this code must pass every vector in +[`test/vectors/args-digest.json`](../test/vectors/args-digest.json), which +`test/args-digest-vectors.test.js` holds to `argsDigestOf`. The vectors cover +key order by UTF-16 code units, string escaping, number formatting (`1.0`, +`-0`, `1e21`, small floats), and the cases where a hand-rolled key-sorted +serializer drifts: an undefined value in an object drops its key, one in an +array becomes `null`, and `NaN` or `Infinity` anywhere means the arguments +have no digest. On arguments that can't be digested, the writer omits +`argsDigest` rather than writing a different digest, because a different +digest would not match the approval the call ran under. + ### Credentials | kind | data | default actor | diff --git a/packages/seats/test/args-digest-vectors.test.js b/packages/seats/test/args-digest-vectors.test.js new file mode 100644 index 0000000000..e597511789 --- /dev/null +++ b/packages/seats/test/args-digest-vectors.test.js @@ -0,0 +1,59 @@ +/** + * test/vectors/args-digest.json is the contract a writer in another language + * passes to produce the same argsDigest as argsDigestOf. This test holds the + * vectors to the code, so neither can change without the other. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { argsDigestOf } from '../src/record/format.js'; + +const FILE = join(dirname(fileURLToPath(import.meta.url)), 'vectors', 'args-digest.json'); +const { _about: about, vectors } = JSON.parse(readFileSync(FILE, 'utf8')); + +/** The file's two tags: {"$undefined":true} and {"$number":"<literal>"}, each an object with one key. */ +const decode = (value) => { + if (Array.isArray(value)) return value.map(decode); + if (value === null || typeof value !== 'object') return value; + const keys = Object.keys(value); + if (keys.length === 1 && keys[0] === '$undefined' && value.$undefined === true) return undefined; + if (keys.length === 1 && keys[0] === '$number' && typeof value.$number === 'string') return Number(value.$number); + return Object.fromEntries(keys.map((key) => [key, decode(value[key])])); +}; + +test('the vector file documents its tags and covers what writers get wrong', () => { + assert.match(about, /\$undefined/); + assert.match(about, /\$number/); + const names = new Set(vectors.map((v) => v.name)); + assert.equal(names.size, vectors.length, 'vector names are distinct'); + for (const v of vectors) { + assert.equal(typeof v.name, 'string'); + assert.ok('args' in v, `${v.name} has args`); + if (v.error === true) { + assert.deepEqual(Object.keys(v).sort(), ['args', 'error', 'name'], `${v.name}: an error vector has no digest`); + } else { + assert.match(v.digest, /^sha256:[0-9a-f]{64}$/, v.name); + assert.equal(typeof v.canonical, 'string', v.name); + } + } + const raw = JSON.stringify(vectors); + for (const tag of ['{"$undefined":true}', '{"$number":"NaN"}', '{"$number":"-0"}', '{"$number":"1e21"}', '{"$number":"1.0"}']) { + assert.ok(raw.includes(tag), `a vector uses ${tag}`); + } + assert.ok(vectors.filter((v) => v.error).length >= 3, 'non-finite and undefined inputs are covered'); +}); + +for (const v of vectors) { + test(`args digest vector: ${v.name}`, () => { + const got = argsDigestOf(decode(v.args)); + if (v.error === true) { + assert.equal(got, undefined, 'no digest: a writer omits argsDigest'); + return; + } + assert.equal(v.digest, `sha256:${createHash('sha256').update(v.canonical, 'utf8').digest('hex')}`, 'digest is sha256 of canonical'); + assert.equal(got, v.digest); + }); +} diff --git a/packages/seats/test/vectors/args-digest.json b/packages/seats/test/vectors/args-digest.json new file mode 100644 index 0000000000..816db7a1c0 --- /dev/null +++ b/packages/seats/test/vectors/args-digest.json @@ -0,0 +1,318 @@ +{ + "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested, and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape.", + "vectors": [ + { + "name": "flat object, keys reordered", + "args": { + "b": 2, + "a": 1 + }, + "canonical": "{\"a\":1,\"b\":2}", + "digest": "sha256:43258cff783fe7036d8a43033f830adfc60ec037382473548ac742b888292777" + }, + { + "name": "nested objects and arrays sort keys at every depth", + "args": { + "z": { + "y": [ + { + "d": 1, + "c": 2 + } + ], + "x": null + }, + "a": { + "c": true, + "b": false + } + }, + "canonical": "{\"a\":{\"b\":false,\"c\":true},\"z\":{\"x\":null,\"y\":[{\"c\":2,\"d\":1}]}}", + "digest": "sha256:e361ba9cd9b9c2b05358b3a26828db49e7e3799a8ab9bdb2aa257247df0b2aab" + }, + { + "name": "keys sort by UTF-16 code units, not code points or UTF-8 bytes", + "args": { + "Ａ": 1, + "😀": 2, + "é": 3, + "z": 4, + "Z": 5 + }, + "canonical": "{\"Z\":5,\"z\":4,\"é\":3,\"😀\":2,\"Ａ\":1}", + "digest": "sha256:8777efd1f0f6d62748dd03f53d88c1228d506e1efd54aafc6e347ee191d15bc5" + }, + { + "name": "empty object", + "args": {}, + "canonical": "{}", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + { + "name": "empty array", + "args": [], + "canonical": "[]", + "digest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "name": "top-level array keeps order", + "args": [ + 3, + 1, + 2 + ], + "canonical": "[3,1,2]", + "digest": "sha256:51bda7ab4e44726cde71fcb6e4b515357059bb6b6dd5146d1fc50f73f11678c6" + }, + { + "name": "deeply nested arrays", + "args": [ + 1, + [ + 2, + [ + 3, + [] + ] + ] + ], + "canonical": "[1,[2,[3,[]]]]", + "digest": "sha256:2ae49150ac570d8d71784d5c84619d21a59e217faf3c632b0ef7a5477b61eb4d" + }, + { + "name": "top-level string", + "args": "git push origin main", + "canonical": "\"git push origin main\"", + "digest": "sha256:0703afb7169909864706a34e00de8cb86207aa3857e72539f34fe6cfeccd1e91" + }, + { + "name": "top-level null", + "args": null, + "canonical": "null", + "digest": "sha256:74234e98afe7498fb5daf1f36ac2d78acc339464f950703b8c019892f982b90b" + }, + { + "name": "booleans and null", + "args": { + "t": true, + "f": false, + "n": null + }, + "canonical": "{\"f\":false,\"n\":null,\"t\":true}", + "digest": "sha256:22e00dc2f7b01420f940fbdbfbdf34fa0667cc6500186495023ba37722cbd05e" + }, + { + "name": "non-ASCII strings are written raw, not escaped", + "args": { + "s": "héllo wörld 日本 😀" + }, + "canonical": "{\"s\":\"héllo wörld 日本 😀\"}", + "digest": "sha256:44832c661e31ea40b67dd594e8b047c2f478fd93007426874dca007289058847" + }, + { + "name": "control characters use short escapes, then \\u00XX lowercase", + "args": { + "s": "\b\f\n\r\t\u0000\u0007\u001f" + }, + "canonical": "{\"s\":\"\\b\\f\\n\\r\\t\\u0000\\u0007\\u001f\"}", + "digest": "sha256:668550beedb0365aa4517dfe85faa784af491a2b29e94205e9b4e2da80298ac7" + }, + { + "name": "quote and backslash are escaped; / < > & DEL U+2028 are not", + "args": { + "s": "\"\\/<>&

" + }, + "canonical": "{\"s\":\"\\\"\\\\/<>&

\"}", + "digest": "sha256:2cd92a4cd6d75d7908756eddfd66f0b4bd361808bc32ea278fcb7dbde1f0eefa" + }, + { + "name": "a lone surrogate is written as a lowercase \\u escape", + "args": { + "s": "\ud800" + }, + "canonical": "{\"s\":\"\\ud800\"}", + "digest": "sha256:d06a70a1ca4d3ac4099cd5f35ecbb551be652247e0950c05790e8f0c58010851" + }, + { + "name": "1.0 is written as 1", + "args": { + "n": { + "$number": "1.0" + } + }, + "canonical": "{\"n\":1}", + "digest": "sha256:2bfd14f43d17fc7cea24e0917a8879b4b2f880b8baeec1b9d90fbaad655e71bd" + }, + { + "name": "-0 is written as 0", + "args": { + "n": { + "$number": "-0" + } + }, + "canonical": "{\"n\":0}", + "digest": "sha256:f3013f933b9fb80ab6d995e7ad9da36f683837ba1d81e950c943d40111eac2f0" + }, + { + "name": "1e20 is written in full", + "args": { + "n": { + "$number": "1e20" + } + }, + "canonical": "{\"n\":100000000000000000000}", + "digest": "sha256:58d2d5b8dd4228ab5775ce84f996718fa19ed49872271e9649b70cebaca26a09" + }, + { + "name": "1e21 switches to exponent form with a plus sign", + "args": { + "n": { + "$number": "1e21" + } + }, + "canonical": "{\"n\":1e+21}", + "digest": "sha256:f1ee2b60ee95a3170fdc07a577e5f3514ced26867443d69da265acadead81007" + }, + { + "name": "0.000001 stays decimal", + "args": { + "n": { + "$number": "0.000001" + } + }, + "canonical": "{\"n\":0.000001}", + "digest": "sha256:28343867a0be00aee19f81aa90cfd6c646878b9303fa15410d16bfd3f8894578" + }, + { + "name": "1e-7 switches to exponent form", + "args": { + "n": { + "$number": "1e-7" + } + }, + "canonical": "{\"n\":1e-7}", + "digest": "sha256:747d6d23b64d1b2d579adb832b44de31c91c875bbef7a8e397f5d183a746b54b" + }, + { + "name": "0.1 + 0.2 keeps the shortest round-trip digits", + "args": { + "n": { + "$number": "0.30000000000000004" + } + }, + "canonical": "{\"n\":0.30000000000000004}", + "digest": "sha256:1d8a8e16d9ee625f1fb5ef505413efa212cc020b272ed0f9b3d3d81acf593fde" + }, + { + "name": "small and large finite extremes", + "args": { + "min": { + "$number": "5e-324" + }, + "max": { + "$number": "1.7976931348623157e308" + } + }, + "canonical": "{\"max\":1.7976931348623157e+308,\"min\":5e-324}", + "digest": "sha256:a49cfc2a701d0651c438486e57d8c53c451fb88c58de9be9be055a6e5f5dd13d" + }, + { + "name": "negative and fractional numbers", + "args": { + "a": { + "$number": "-1.5" + }, + "b": { + "$number": "-123" + }, + "c": { + "$number": "3.14159" + } + }, + "canonical": "{\"a\":-1.5,\"b\":-123,\"c\":3.14159}", + "digest": "sha256:fd4545080eaf9490b1e10de2680fc77a1a90886c9c494a05f1144cdb33d38ae0" + }, + { + "name": "an integer past 2^53 is read as an IEEE double first", + "args": { + "n": { + "$number": "9007199254740993" + } + }, + "canonical": "{\"n\":9007199254740992}", + "digest": "sha256:66c87d9cb3014e05a11baa97df62282d89d425f22ee15816577c84534e2ef1bb" + }, + { + "name": "undefined in an object drops the key", + "args": { + "a": 1, + "b": { + "$undefined": true + } + }, + "canonical": "{\"a\":1}", + "digest": "sha256:015abd7f5cc57a2dd94b7590f04ad8084273905ee33ec5cebeae62276a97f862" + }, + { + "name": "undefined nested in an object drops only that key", + "args": { + "a": { + "b": { + "$undefined": true + }, + "c": 2 + } + }, + "canonical": "{\"a\":{\"c\":2}}", + "digest": "sha256:793fe66a2d4239e3e64774ce7ee90c6e1931ff33712ff05a17956a52e577c753" + }, + { + "name": "undefined in an array becomes null", + "args": [ + 1, + { + "$undefined": true + }, + 3 + ], + "canonical": "[1,null,3]", + "digest": "sha256:7db5b36a1553a4e31914df4beb60752bd07320e3056ac1d3790660c4cad6a173" + }, + { + "name": "NaN anywhere means no digest", + "args": { + "a": 1, + "b": { + "$number": "NaN" + } + }, + "error": true + }, + { + "name": "Infinity nested in an array means no digest", + "args": { + "a": [ + 1, + { + "$number": "Infinity" + } + ] + }, + "error": true + }, + { + "name": "-Infinity means no digest", + "args": { + "$number": "-Infinity" + }, + "error": true + }, + { + "name": "undefined arguments mean no digest", + "args": { + "$undefined": true + }, + "error": true + } + ] +} From 1e82819c42db0b0428f397b5522dad1ac1721f0b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:20:26 -0700 Subject: [PATCH 08/20] feat(seats): no args digest on a lone surrogate; the tool span carries onBehalfOf argsDigestOf now returns undefined when any string or key in the arguments holds a lone surrogate, as RFC 8785 requires, so a writer omits argsDigest rather than digesting bytes another language's JSON decoder would not reproduce. Only the digest refuses: canonicalLine still records agent text cut mid-pair, as a \u escape. The lone surrogate vector becomes four no-digest vectors (high, low nested in an array, reversed pair, in a key); 34 vectors in all. The execute_tool span now carries seats.actor.on_behalf_of.id and .role from the tool.call actor, so ADR-025 D18's pair sits on the action itself as well as on its parent attempt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 11 +++++-- packages/seats/src/record/format.js | 23 +++++++++---- packages/seats/src/record/otel.js | 1 + .../seats/test/args-digest-vectors.test.js | 1 + packages/seats/test/record-format.test.js | 15 +++++++++ packages/seats/test/record-otel.test.js | 7 ++-- packages/seats/test/vectors/args-digest.json | 33 ++++++++++++++++--- 7 files changed, 75 insertions(+), 16 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index b1655d9e23..5255d02964 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -295,9 +295,12 @@ key order by UTF-16 code units, string escaping, number formatting (`1.0`, `-0`, `1e21`, small floats), and the cases where a hand-rolled key-sorted serializer drifts: an undefined value in an object drops its key, one in an array becomes `null`, and `NaN` or `Infinity` anywhere means the arguments -have no digest. On arguments that can't be digested, the writer omits -`argsDigest` rather than writing a different digest, because a different -digest would not match the approval the call ran under. +have no digest. So does a lone surrogate (half of a UTF-16 pair) in any +string or key, as RFC 8785 requires. Only the digest refuses it: a line +still records agent text holding one, as a `\u` escape, because a stream +cut mid-pair must not lose the chunk. On arguments that can't be digested, +the writer omits `argsDigest` rather than writing a different digest, +because a different digest would not match the approval the call ran under. ### Credentials @@ -493,6 +496,8 @@ to that commit. `error.type` is Stable. `seats.tool.surface` and `seats.tool.args_digest`; - on every event, `seats.seq` and `seats.actor.type`, `.id`, `.owner`, `.name`, `.on_behalf_of.id` and `.on_behalf_of.role`; + - on the tool span, `seats.actor.on_behalf_of.id` and `.role` from the + `tool.call` actor, so the pair sits on the action itself; - the event's data, flattened as `seats.<path>` with camelCase segments in snake_case. For example, `decidedBy` becomes `seats.decided_by`, and `scope.connectionId` becomes `seats.scope.connection_id`, and diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 0e5fa51588..c5efba4aa5 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -86,20 +86,28 @@ const isTimestamp = (value) => typeof value === 'string' && RFC3339_UTC.test(val * RFC 8785 (JCS) serialization. For the JSON values a record holds, JCS is * JSON.stringify with object keys sorted by UTF-16 code units, which is what * Array.prototype.sort does by default. + * + * `strict` also refuses a lone surrogate in any string or key, as RFC 8785 + * requires. Lines are not strict: agent text from a stream cut mid-pair must + * still record, and JSON.stringify writes the lone half as a \u escape. */ -const canonical = (value) => { - if (value !== null && typeof value === 'object' && typeof value.toJSON === 'function') return canonical(value.toJSON()); +const canonicalString = (text, strict) => { + if (strict && !text.isWellFormed()) fail('strings must be well-formed UTF-16 (no lone surrogates)'); + return JSON.stringify(text); +}; +const canonical = (value, strict = false) => { + if (value !== null && typeof value === 'object' && typeof value.toJSON === 'function') return canonical(value.toJSON(), strict); if (value === null) return 'null'; switch (typeof value) { - case 'string': return JSON.stringify(value); + case 'string': return canonicalString(value, strict); case 'boolean': return value ? 'true' : 'false'; case 'number': if (!Number.isFinite(value)) fail('numbers must be finite'); return JSON.stringify(value); case 'object': - if (Array.isArray(value)) return `[${value.map((item) => (item === undefined ? 'null' : canonical(item))).join(',')}]`; + if (Array.isArray(value)) return `[${value.map((item) => (item === undefined ? 'null' : canonical(item, strict))).join(',')}]`; return `{${Object.keys(value).filter((key) => value[key] !== undefined).sort() - .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + .map((key) => `${canonicalString(key, strict)}:${canonical(value[key], strict)}`).join(',')}}`; default: return fail(`cannot serialize a ${typeof value}`); } }; @@ -118,12 +126,13 @@ export const hashLine = (line) => createHash('sha256').update(String(line), 'utf * lowercase hex SHA-256 of the arguments' RFC 8785 form. The same arguments * give the same digest wherever they are seen, so a reader can match a * tool.call to the permission entries that approved it. Returns undefined - * when there are no arguments, or they are not JSON (a function, NaN). + * when there are no arguments, or they are not JSON (a function, NaN, a + * string or key holding a lone surrogate): a writer then omits argsDigest. */ const ARGS_DIGEST = /^sha256:[0-9a-f]{64}$/; export const argsDigestOf = (args) => { if (args === undefined) return undefined; - try { return `sha256:${hashLine(canonical(args))}`; } catch { return undefined; } + try { return `sha256:${hashLine(canonical(args, true))}`; } catch { return undefined; } }; export const isArgsDigest = (value) => typeof value === 'string' && ARGS_DIGEST.test(value); diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js index 0d031d2a1a..cf97eff523 100644 --- a/packages/seats/src/record/otel.js +++ b/packages/seats/src/record/otel.js @@ -170,6 +170,7 @@ const spansOfRun = (entries) => { ['gen_ai.operation.name', 'execute_tool'], ['gen_ai.tool.name', toolName], ['gen_ai.tool.call.id', data.toolCallId], ['gen_ai.agent.name', attempt?.agentName], + ['seats.actor.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor?.onBehalfOf?.role], ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], ['seats.tool.args_digest', data.argsDigest], ]), }); diff --git a/packages/seats/test/args-digest-vectors.test.js b/packages/seats/test/args-digest-vectors.test.js index e597511789..2d0b2c3a91 100644 --- a/packages/seats/test/args-digest-vectors.test.js +++ b/packages/seats/test/args-digest-vectors.test.js @@ -44,6 +44,7 @@ test('the vector file documents its tags and covers what writers get wrong', () assert.ok(raw.includes(tag), `a vector uses ${tag}`); } assert.ok(vectors.filter((v) => v.error).length >= 3, 'non-finite and undefined inputs are covered'); + assert.ok(vectors.some((v) => v.error && /lone surrogate/.test(v.name)), 'a lone surrogate is a no-digest case'); }); for (const v of vectors) { diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 47340612f0..0450a94c1a 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -298,6 +298,8 @@ test('argsDigest is sha256 of the canonical args, and anything else is refused, assert.equal(argsDigestOf({ a: [1, 'x'], b: 2 }), digest, 'key order does not change it'); assert.equal(argsDigestOf(undefined), undefined); assert.equal(argsDigestOf({ n: Number.NaN }), undefined); + assert.equal(argsDigestOf({ s: '\ud800' }), undefined, 'a lone surrogate has no digest (RFC 8785)'); + assert.equal(argsDigestOf({ ['k\udc00']: 1 }), undefined, 'in a key too'); const call = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', argsDigest } }); const decided = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data: { outcome: 'cancelled', toolCallId: 't', argsDigest } }); for (const write of [call, decided, (d) => request(askData({ argsDigest: d }))]) { @@ -362,3 +364,16 @@ test('actor.onBehalfOf is additive: a record written without it is byte-identica assert.ok(!l1.includes('onBehalfOf') && !l2.includes('onBehalfOf')); assert.deepEqual(verifyRecord(`${l1}\n${l2}\n`).errors, []); }); + +test('a lone surrogate in agent text still records: only the args digest refuses it', () => { + // A stream cut mid-pair leaves half a surrogate pair in a chunk. + const chunk = `cut here ${'\ud83d'}`; + const first = appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.message_chunk', data: { text: chunk } }, { runActor: AGENT }); + assert.ok(first.includes('cut here \\ud83d'), 'the lone half is written as a \\u escape'); + const second = appendEntry(first, { + ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 'tc-1', title: chunk, argsDigest: argsDigestOf({ q: 'ok' }) }, + }, { runActor: AGENT }); + assert.deepEqual(verifyRecord(`${first}\n${second}\n`).errors, []); + assert.equal(readRecord(`${first}\n`)[0].data.text, chunk); + assert.equal(argsDigestOf({ q: chunk }), undefined); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index aa4cd6661e..18da4fe10e 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -154,7 +154,7 @@ test('decision.ruled joins the tool span of the request on its card, else the sp assert.ok(again.events.some((e) => e.name === 'decision.ruled')); }); -test('actor.onBehalfOf maps beside actor.owner: on every event, and on the attempt span', () => { +test('actor.onBehalfOf maps beside actor.owner: on every event, the attempt span and the tool span', () => { const base = { v: 1, schema: 'seats.record', runId: 'run-obo-0001', prev: null }; const actor = { type: 'agent', id: 'u-agent-ana', owner: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } }; const entries = [ @@ -171,7 +171,10 @@ test('actor.onBehalfOf maps beside actor.owner: on every event, and on the attem assert.equal(attrOf(attempt, 'seats.agent.owner').stringValue, 'u-ana'); assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.id').stringValue, 'u-ben'); assert.equal(attrOf(attempt, 'seats.agent.on_behalf_of.role').stringValue, 'requester'); - const asked = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue').events.find((e) => e.name === 'permission.requested'); + const tool = spans.find((s) => s.name === 'execute_tool sentry.resolve_issue'); + assert.equal(attrOf(tool, 'seats.actor.on_behalf_of.id').stringValue, 'u-ben', 'the pair is on the action itself'); + assert.equal(attrOf(tool, 'seats.actor.on_behalf_of.role').stringValue, 'requester'); + const asked = tool.events.find((e) => e.name === 'permission.requested'); assert.equal(attrOf(asked, 'seats.actor.owner').stringValue, 'u-ana'); assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.id').stringValue, 'u-ben'); assert.equal(attrOf(asked, 'seats.actor.on_behalf_of.role').stringValue, 'requester'); diff --git a/packages/seats/test/vectors/args-digest.json b/packages/seats/test/vectors/args-digest.json index 816db7a1c0..c1a903aea5 100644 --- a/packages/seats/test/vectors/args-digest.json +++ b/packages/seats/test/vectors/args-digest.json @@ -1,5 +1,5 @@ { - "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested, and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape.", + "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested (a non-finite number, undefined arguments, or a lone surrogate in any string or key), and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape.", "vectors": [ { "name": "flat object, keys reordered", @@ -126,12 +126,37 @@ "digest": "sha256:2cd92a4cd6d75d7908756eddfd66f0b4bd361808bc32ea278fcb7dbde1f0eefa" }, { - "name": "a lone surrogate is written as a lowercase \\u escape", + "name": "a lone high surrogate in a string means no digest", "args": { "s": "\ud800" }, - "canonical": "{\"s\":\"\\ud800\"}", - "digest": "sha256:d06a70a1ca4d3ac4099cd5f35ecbb551be652247e0950c05790e8f0c58010851" + "error": true + }, + { + "name": "a lone low surrogate nested in an array means no digest", + "args": { + "a": [ + "ok", + [ + "x\udc00y" + ] + ] + }, + "error": true + }, + { + "name": "a surrogate pair in the wrong order means no digest", + "args": { + "s": "\ude00\ud83d" + }, + "error": true + }, + { + "name": "a lone surrogate in a key means no digest", + "args": { + "k\ud800": 1 + }, + "error": true }, { "name": "1.0 is written as 1", From 50437c912bddaa102b2cf1e08cbd3c0295eab611 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:28:15 -0700 Subject: [PATCH 09/20] feat(seats): record egress.allowed and egress.denied; vectors gate other-language writers The egress fence emits egress.allowed {host, port} and egress.denied {host, port, reason}, and the record's emit threw on both. They are now v1 kinds, group egress, default actor system. host is a lowercase DNS name, an IP literal or "[redacted]"; port is 1-65535; reason is the closed set EGRESS_DENIED_REASONS (bad_request, not_connect, ip_literal, not_allowed, unresolved, private_address, unreachable). host and port are null together and only for bad_request or not_connect; no other field is allowed. Both export to OTLP as events on the current span. Vectors: the past-2^53 vector now holds the JS-parsed double (9007199254740992) and a large-integer formatting vector joins it; U+2028/U+2029 get their own vector; _about says a non-JS writer parses numbers as IEEE 754 doubles first. 36 vectors. record-v1.md now states the vectors file as the conformance gate: a writer in any language that fails it does not ship, and a Go writer runs its tests with -race in CI. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 75 +++++++++++++++----- packages/seats/src/record/format.js | 35 +++++++++ packages/seats/test/record-format.test.js | 51 ++++++++++++- packages/seats/test/record-otel.test.js | 25 +++++++ packages/seats/test/record-store.test.js | 11 +++ packages/seats/test/vectors/args-digest.json | 26 +++++-- 6 files changed, 199 insertions(+), 24 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 5255d02964..5a3841efd0 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -287,20 +287,28 @@ request, and the OTLP export places it there. - A digest of low-entropy arguments can be confirmed by guessing them. It hides the values from a casual reader, not from someone who can guess them. -**Writers in other languages.** A writer that computes `argsDigest` outside -this code must pass every vector in -[`test/vectors/args-digest.json`](../test/vectors/args-digest.json), which -`test/args-digest-vectors.test.js` holds to `argsDigestOf`. The vectors cover -key order by UTF-16 code units, string escaping, number formatting (`1.0`, -`-0`, `1e21`, small floats), and the cases where a hand-rolled key-sorted -serializer drifts: an undefined value in an object drops its key, one in an -array becomes `null`, and `NaN` or `Infinity` anywhere means the arguments -have no digest. So does a lone surrogate (half of a UTF-16 pair) in any -string or key, as RFC 8785 requires. Only the digest refuses it: a line -still records agent text holding one, as a `\u` escape, because a stream -cut mid-pair must not lose the chunk. On arguments that can't be digested, -the writer omits `argsDigest` rather than writing a different digest, -because a different digest would not match the approval the call ran under. +**Conformance gate for writers in other languages.** Every record writer, +in any language, must pass every vector in +[`test/vectors/args-digest.json`](../test/vectors/args-digest.json), and a +writer that fails it does not ship. `test/args-digest-vectors.test.js` holds +the file to `argsDigestOf`, so the file is the definition, not a sample. The +vectors cover: +- key order by UTF-16 code units; +- string escaping, with U+2028 and U+2029 written raw; +- number formatting: `1.0` is `1`, `-0` is `0`, `1e21`, small floats, and an + integer past 2^53, which a writer parses as an IEEE 754 double first, so + `9007199254740993` on the wire is written `9007199254740992`; +- the cases where a hand-rolled key-sorted serializer drifts: an undefined + value in an object drops its key, one in an array becomes `null`, and `NaN` + or `Infinity` anywhere means the arguments have no digest. + +A lone surrogate (half of a UTF-16 pair) in any string or key also means no +digest, as RFC 8785 requires. Only the digest refuses it: a line still +records agent text holding one, as a `\u` escape, because a stream cut +mid-pair must not lose the chunk. On arguments that can't be digested, a +writer omits `argsDigest` rather than writing a different digest, because a +different digest would not match the approval the call ran under. A writer +built in Go runs its tests with `-race` in CI. ### Credentials @@ -313,6 +321,36 @@ These cover the environment credentials a run is started with. Values never appear here: `NAME` is an environment variable name matching `[A-Z_][A-Z0-9_]*`. +### Egress + +A sandboxed run reaches the network only through the egress proxy, which +admits the hosts its grant names, plus its model provider. These lines +record what the proxy did. + +| kind | data | default actor | +|---|---|---| +| `egress.allowed` | `{host, port}`: a tunnel opened | `system` | +| `egress.denied` | `{host, port, reason}`: a request was refused | `system` | + +- `host` is a lowercase DNS name, an IP literal (IPv6 without brackets), or + the literal string `"[redacted]"` when the requested name held one of the + run's credentials. +- `port` is an integer from 1 to 65535. +- `reason` is one of: + - `bad_request`: the request could not be parsed; + - `not_connect`: a method other than `CONNECT`; + - `ip_literal`: the target was an address, not a name; + - `not_allowed`: no grant entry names the host and port; + - `unresolved`: the name did not resolve; + - `private_address`: it resolved to a non-public address that no exact + entry names; + - `unreachable`: the connection failed. +- `host` and `port` are `null` together, and only on a malformed request + (`bad_request`, or `not_connect` when the target can't be read). Every + other line names both. +- No other field is allowed, so no header, body or resolved address is + recorded. + ### Result | kind | data | default actor | @@ -383,8 +421,9 @@ What's new or changed in `EVENT_TYPES`: - The verifier checks the envelope and the chain, never data, so those lines still verify. - A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`. - Lines without it are unchanged. + A later revision, also at `v: 1`, added the optional `actor.onBehalfOf` + and the kinds `egress.allowed` and `egress.denied`. Lines without them + are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -436,8 +475,8 @@ operates it. that carried its `decisionCardId`, and otherwise on the span current at the time. - **Every other kind is an event on the span current at the time.** That - covers grants, credentials, claims, `task.done`, results, liveness and - `tool.update`. + covers grants, credentials, claims, `task.done`, results, liveness, egress + and `tool.update`. An egress denial is an event, not an error status. - **Span ids** are the first 8 bytes of SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to the entry that opens the span. The run span uses `0`. Exporting the same diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index c5efba4aa5..78829044d3 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -5,6 +5,7 @@ * the store, so another tool can vendor it alone. */ import { createHash } from 'node:crypto'; +import { isIP } from 'node:net'; import { FAILURE_KINDS } from '../failures.js'; export const RECORD_VERSION = 1; @@ -24,6 +25,12 @@ export const RESULT_STATUSES = Object.freeze(['done', 'partial', 'blocked', 'fai // Whose request an agent acts on: 'requester' sent the message that started // the run, 'assigner' assigned its task. Absent when no human asked. export const ON_BEHALF_ROLES = Object.freeze(['requester', 'assigner']); +// Why the egress proxy refused a request. The first two can leave host and port null. +export const EGRESS_DENIED_REASONS = Object.freeze([ + 'bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable', +]); +const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); +export const REDACTED = '[redacted]'; /** The actors a writer records for its own bookkeeping, and for the board. */ export const SYSTEM_ACTOR = Object.freeze({ type: 'system', id: 'seats' }); @@ -59,6 +66,8 @@ export const KINDS = Object.freeze({ 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), 'liveness.recovery': Object.freeze({ group: 'activity', actor: 'system' }), 'process.exited': Object.freeze({ group: 'activity', actor: 'system' }), + 'egress.allowed': Object.freeze({ group: 'egress', actor: 'system' }), + 'egress.denied': Object.freeze({ group: 'egress', actor: 'system' }), }); const TRACE_TO_KIND = new Map(Object.entries(KINDS).filter(([, spec]) => spec.trace).map(([kind, spec]) => [spec.trace, kind])); @@ -188,6 +197,21 @@ const userIdOrPolicy = (value, where) => { const optionalArgsDigest = (data, kind) => { if (data.argsDigest !== undefined && !isArgsDigest(data.argsDigest)) fail(`${kind}.argsDigest must be sha256:<64 hex>, a digest of the arguments, never the arguments`); }; +// A host as the egress proxy traces it: a lowercase DNS name, an IP literal +// (IPv6 without brackets), or [redacted] when it held a run credential. +const DNS_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; +const isEgressHost = (host) => typeof host === 'string' && (host === REDACTED || isIP(host) !== 0 + || (host.length <= 253 && host.split('.').every((label) => DNS_LABEL.test(label)))); +const isPort = (port) => Number.isInteger(port) && port >= 1 && port <= 65535; +const checkEgressTarget = (data, kind, { nullable }) => { + if (data.host === null && data.port === null && nullable) return; + if (data.host === null || data.port === null) { + fail(nullable ? `${kind}: host and port are null together, or both set` : `${kind} needs a host and a port`); + } + if (!isEgressHost(data.host)) fail(`${kind}.host must be a lowercase DNS name, an IP literal or "${REDACTED}"`); + if (!isPort(data.port)) fail(`${kind}.port must be an integer from 1 to 65535`); +}; + const checkRole = (role, where) => { if (!DECIDED_BY_ROLES.includes(role)) fail(`${where} must be one of ${DECIDED_BY_ROLES.join(', ')}`); }; @@ -325,6 +349,17 @@ const CHECKS = { } return data; }, + 'egress.allowed': (data) => { + refuseUnknownKeys(data, ['host', 'port'], 'egress.allowed.'); + checkEgressTarget(data, 'egress.allowed', { nullable: false }); + return data; + }, + 'egress.denied': (data) => { + refuseUnknownKeys(data, ['host', 'port', 'reason'], 'egress.denied.'); + if (!EGRESS_DENIED_REASONS.includes(data.reason)) fail(`egress.denied.reason must be one of ${EGRESS_DENIED_REASONS.join(', ')}`); + checkEgressTarget(data, 'egress.denied', { nullable: EGRESS_TARGETLESS.has(data.reason) }); + return data; + }, 'result.reported': (data) => { if (!RESULT_STATUSES.includes(data.status)) fail(`result.reported.status must be one of ${RESULT_STATUSES.join(', ')}`); if (data.refs !== undefined && !(Array.isArray(data.refs) && data.refs.every((ref) => isPlainObject(ref) && isText(ref.type) && isText(ref.id)))) { diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 0450a94c1a..839679261c 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { - appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, KINDS, ON_BEHALF_ROLES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -377,3 +377,52 @@ test('a lone surrogate in agent text still records: only the args digest refuses assert.equal(readRecord(`${first}\n`)[0].data.text, chunk); assert.equal(argsDigestOf({ q: chunk }), undefined); }); + +const egress = (kind, data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind, data }, { runActor: AGENT })); + +test('egress.allowed: {host, port}, written by the system', () => { + assert.deepEqual(KINDS['egress.allowed'], { group: 'egress', actor: 'system' }); + const ok = egress('egress.allowed', { host: 'api.sentry.io', port: 443 }); + assert.deepEqual(ok.data, { host: 'api.sentry.io', port: 443 }); + assert.deepEqual(ok.actor, { type: 'system', id: 'seats' }); + // host: a lowercase DNS name, an IP literal, or [redacted]; anything else is refused. + for (const host of ['localhost', '[redacted]', '203.0.113.7', '2001:db8::1']) egress('egress.allowed', { host, port: 443 }); + for (const host of ['API.sentry.io', 'https://api.sentry.io', 'user@api.sentry.io', 'a b.io', '-x.io', '', '[REDACTED]', 42]) { + assert.throws(() => egress('egress.allowed', { host, port: 443 }), /egress.allowed.host/, `host ${JSON.stringify(host)}`); + } + assert.throws(() => egress('egress.allowed', { host: `${'a'.repeat(60)}.`.repeat(5), port: 443 }), /egress.allowed.host/, 'over 253 characters'); + // port: an integer from 1 to 65535. + egress('egress.allowed', { host: 'api.sentry.io', port: 65535 }); + for (const port of [0, 65536, 443.5, '443', -1]) assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io', port }), /egress.allowed.port/, `port ${port}`); + // An opened tunnel always names its target: no nulls, no missing fields. + assert.throws(() => egress('egress.allowed', { host: null, port: null }), /needs a host and a port/); + assert.throws(() => egress('egress.allowed', { port: 443 }), /egress.allowed.host/); + assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io' }), /egress.allowed.port/); + // No other field: a header or an address can't ride in. + assert.throws(() => egress('egress.allowed', { host: 'api.sentry.io', port: 443, address: '203.0.113.7' }), /unknown field egress.allowed.address/); +}); + +test('egress.denied: {host, port, reason}, the reason from a closed set, null target only on a malformed request', () => { + assert.deepEqual(KINDS['egress.denied'], { group: 'egress', actor: 'system' }); + assert.deepEqual(EGRESS_DENIED_REASONS, ['bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable']); + const denied = egress('egress.denied', { host: 'evil.example', port: 443, reason: 'not_allowed' }); + assert.deepEqual(denied.data, { host: 'evil.example', port: 443, reason: 'not_allowed' }); + assert.deepEqual(denied.actor, { type: 'system', id: 'seats' }); + // reason: every listed reason is accepted; anything else is refused. + for (const reason of EGRESS_DENIED_REASONS) egress('egress.denied', { host: '10.0.0.1', port: 80, reason }); + for (const reason of ['forbidden', undefined, null, 'NOT_ALLOWED']) { + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 443, reason }), /egress.denied.reason/, `reason ${reason}`); + } + // host and port may be null, together, when the request was malformed. + for (const reason of ['bad_request', 'not_connect']) { + assert.deepEqual(egress('egress.denied', { host: null, port: null, reason }).data, { host: null, port: null, reason }); + } + assert.throws(() => egress('egress.denied', { host: null, port: null, reason: 'not_allowed' }), /needs a host and a port/, 'a parsed target is never null'); + assert.throws(() => egress('egress.denied', { host: null, port: 443, reason: 'bad_request' }), /null together/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: null, reason: 'not_connect' }), /null together/); + // host and port follow the egress.allowed rules when set. + egress('egress.denied', { host: '[redacted]', port: 443, reason: 'not_allowed' }); + assert.throws(() => egress('egress.denied', { host: 'Evil.example', port: 443, reason: 'not_allowed' }), /egress.denied.host/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 70000, reason: 'not_allowed' }), /egress.denied.port/); + assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 443, reason: 'not_allowed', status: 403 }), /unknown field egress.denied.status/); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index 18da4fe10e..8553f026e8 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -182,3 +182,28 @@ test('actor.onBehalfOf maps beside actor.owner: on every event, the attempt span const plain = spansOf(toOtlpJson(entries.map((e) => (e.actor.type === 'agent' ? { ...e, actor: { type: 'agent', id: 'u-agent-ana', owner: 'u-ana' } } : e)))); assert.ok(!JSON.stringify(plain).includes('on_behalf_of')); }); + +test('egress entries are events on the span current at the time', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-egr-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'egress.denied', data: { host: null, port: null, reason: 'bad_request' } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', actor: { type: 'agent', id: 'u-agent-ana' }, data: { attempt: 1 } }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'egress.allowed', data: { host: 'api.sentry.io', port: 443 } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'egress.denied', data: { host: '[redacted]', port: 443, reason: 'not_allowed' } }, + { ...base, seq: 5, ts: '2026-10-25T12:00:04.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const run = spans.find((s) => s.name === 'seats.run'); + const attempt = spans.find((s) => s.name.startsWith('invoke_agent')); + const early = run.events.find((e) => e.name === 'egress.denied'); + assert.equal(attrOf(early, 'seats.reason').stringValue, 'bad_request'); + assert.equal(attrOf(early, 'seats.host'), undefined, 'a null host is left out'); + assert.equal(attrOf(early, 'seats.actor.id').stringValue, 'seats'); + const allowed = attempt.events.find((e) => e.name === 'egress.allowed'); + assert.equal(attrOf(allowed, 'seats.host').stringValue, 'api.sentry.io'); + assert.equal(attrOf(allowed, 'seats.port').intValue, '443'); + const denied = attempt.events.find((e) => e.name === 'egress.denied'); + assert.equal(attrOf(denied, 'seats.host').stringValue, '[redacted]'); + assert.equal(attrOf(denied, 'seats.reason').stringValue, 'not_allowed'); + assert.equal(attrOf(attempt, 'error.type'), undefined, 'a denial is an event, not an error status'); +}); diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js index 40ab9e7c55..8e133aaa10 100644 --- a/packages/seats/test/record-store.test.js +++ b/packages/seats/test/record-store.test.js @@ -162,3 +162,14 @@ test('the store writes actor.onBehalfOf through emit, and refuses it on a non-ag assert.throws(() => store.startRun({ runId: 'run-st-obo2', actor: { type: 'human', id: 'u-ana', onBehalfOf: { id: 'u-ben', role: 'requester' } } }), /agent actors only/); assert.throws(() => store.startRun({ runId: 'run-st-obo3', actor: { ...AGENT, onBehalfOf: { id: 'u-ben', role: 'boss' } } }), /onBehalfOf.role/); })); + +test('the store records the egress proxy\'s trace types', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-egr1', actor: AGENT }); + run.emit('egress.allowed', { host: 'api.sentry.io', port: 443 }); + run.emit('egress.denied', { host: null, port: null, reason: 'bad_request' }); + run.emit('egress.denied', { host: '[redacted]', port: 443, reason: 'not_allowed' }); + assert.throws(() => run.emit('egress.denied', { host: 'x.io', port: 443, reason: 'nope' }), /egress.denied.reason/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-egr1').map((e) => e.kind), ['egress.allowed', 'egress.denied', 'egress.denied', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-egr1'), 'utf8')).errors, []); +})); diff --git a/packages/seats/test/vectors/args-digest.json b/packages/seats/test/vectors/args-digest.json index c1a903aea5..6640506027 100644 --- a/packages/seats/test/vectors/args-digest.json +++ b/packages/seats/test/vectors/args-digest.json @@ -1,5 +1,5 @@ { - "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested (a non-finite number, undefined arguments, or a lone surrogate in any string or key), and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape.", + "_about": "Test vectors for argsDigestOf (src/record/format.js; docs/record-v1.md, Tool activity). Each vector is {name, args, canonical, digest} or {name, args, error: true}. digest is \"sha256:\" + lowercase hex SHA-256 of canonical, the RFC 8785 bytes of args. error: true means the arguments cannot be digested (a non-finite number, undefined arguments, or a lone surrogate in any string or key), and a writer omits argsDigest. JSON cannot hold every input, so args uses two tags, each an object with exactly one key: {\"$undefined\":true} is an undefined value, and {\"$number\":\"<literal>\"} is the number that literal parses to as an IEEE 754 double (e.g. \"1.0\", \"-0\", \"1e21\", \"NaN\", \"Infinity\"). Decode the tags before digesting; no vector holds a real argument of either shape. A writer outside JavaScript must parse every JSON number as an IEEE 754 double before serializing it (as JSON.parse does), then print its shortest round-trip form as ECMAScript does: 9007199254740993 in the input is the double 9007199254740992, and is written that way. Strings are UTF-8 with U+2028 and U+2029 raw, not escaped.", "vectors": [ { "name": "flat object, keys reordered", @@ -125,6 +125,14 @@ "canonical": "{\"s\":\"\\\"\\\\/<>&

\"}", "digest": "sha256:2cd92a4cd6d75d7908756eddfd66f0b4bd361808bc32ea278fcb7dbde1f0eefa" }, + { + "name": "U+2028 and U+2029 are written raw, never escaped", + "args": { + "s": "a
b
c" + }, + "canonical": "{\"s\":\"a
b
c\"}", + "digest": "sha256:5b2dc062ac2d05c4b51a388dd0e99e9b161c6c6f0fefeef39629890a42902b84" + }, { "name": "a lone high surrogate in a string means no digest", "args": { @@ -258,15 +266,23 @@ "digest": "sha256:fd4545080eaf9490b1e10de2680fc77a1a90886c9c494a05f1144cdb33d38ae0" }, { - "name": "an integer past 2^53 is read as an IEEE double first", + "name": "an integer past 2^53: 9007199254740993 on the wire is the double 9007199254740992", "args": { - "n": { - "$number": "9007199254740993" - } + "n": 9007199254740992 }, "canonical": "{\"n\":9007199254740992}", "digest": "sha256:66c87d9cb3014e05a11baa97df62282d89d425f22ee15816577c84534e2ef1bb" }, + { + "name": "large integers print their shortest round-trip digits, then zeros: 2^60 is 1152921504606847000", + "args": [ + 9007199254740994, + 1152921504606847000, + 10000000000000000 + ], + "canonical": "[9007199254740994,1152921504606847000,10000000000000000]", + "digest": "sha256:0e2b45d1064af2a28028209676fbd29080eca67ae85a2ad6582eb0efec521429" + }, { "name": "undefined in an object drops the key", "args": { From 274d6d303cfcf76588527bf0856e98951985e798 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:30:41 -0700 Subject: [PATCH 10/20] feat(seats): record sandbox.applied as a v1 kind The sandbox lineage (f41f7235e) emits sandbox.applied {sandbox, reason?}, and the record's emit threw on it. It is now a v1 kind in a new sandbox group (it records how the process was confined, not what an attempt did), default actor system. sandbox is the closed set SANDBOXES (seatbelt, bwrap, none); reason is a non-empty string, set only beside none and optional there, since the emitter drops an empty one; no other field. It exports to OTLP as an event on the current span, like every other non-span kind. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 23 +++++++++++++++++++---- packages/seats/src/record/format.js | 12 ++++++++++++ packages/seats/test/record-format.test.js | 21 ++++++++++++++++++++- packages/seats/test/record-otel.test.js | 17 +++++++++++++++++ packages/seats/test/record-store.test.js | 9 +++++++++ 5 files changed, 77 insertions(+), 5 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 5a3841efd0..97c21a0e21 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -321,6 +321,21 @@ These cover the environment credentials a run is started with. Values never appear here: `NAME` is an environment variable name matching `[A-Z_][A-Z0-9_]*`. +### Sandbox + +| kind | data | default actor | +|---|---|---| +| `sandbox.applied` | `{sandbox, reason?}` | `system` | + +- `sandbox` is the OS sandbox the run's process was started in: `seatbelt` + (macOS), `bwrap` (Linux), or `none`. +- `reason` says why there was none. It is set only when `sandbox` is `none`, + and is a non-empty string. A writer may leave it out when it has no + reason to give. +- No other field is allowed. +- It is its own group, not part of Attempt: it records how the process was + confined, not what the attempt did or how it ended. + ### Egress A sandboxed run reaches the network only through the egress proxy, which @@ -422,8 +437,8 @@ What's new or changed in `EVENT_TYPES`: lines still verify. A later revision, also at `v: 1`, added the optional `actor.onBehalfOf` - and the kinds `egress.allowed` and `egress.denied`. Lines without them - are unchanged. + and the kinds `sandbox.applied`, `egress.allowed` and `egress.denied`. + Lines without them are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -475,8 +490,8 @@ operates it. that carried its `decisionCardId`, and otherwise on the span current at the time. - **Every other kind is an event on the span current at the time.** That - covers grants, credentials, claims, `task.done`, results, liveness, egress - and `tool.update`. An egress denial is an event, not an error status. + covers grants, credentials, claims, `task.done`, results, liveness, + `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. - **Span ids** are the first 8 bytes of SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to the entry that opens the span. The run span uses `0`. Exporting the same diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 78829044d3..1e70e095b7 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -30,6 +30,7 @@ export const EGRESS_DENIED_REASONS = Object.freeze([ 'bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable', ]); const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); +export const SANDBOXES = Object.freeze(['seatbelt', 'bwrap', 'none']); export const REDACTED = '[redacted]'; /** The actors a writer records for its own bookkeeping, and for the board. */ @@ -66,6 +67,8 @@ export const KINDS = Object.freeze({ 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), 'liveness.recovery': Object.freeze({ group: 'activity', actor: 'system' }), 'process.exited': Object.freeze({ group: 'activity', actor: 'system' }), + // The run's confinement, recorded once per spawn, beside its egress. + 'sandbox.applied': Object.freeze({ group: 'sandbox', actor: 'system' }), 'egress.allowed': Object.freeze({ group: 'egress', actor: 'system' }), 'egress.denied': Object.freeze({ group: 'egress', actor: 'system' }), }); @@ -349,6 +352,15 @@ const CHECKS = { } return data; }, + 'sandbox.applied': (data) => { + refuseUnknownKeys(data, ['sandbox', 'reason'], 'sandbox.applied.'); + if (!SANDBOXES.includes(data.sandbox)) fail(`sandbox.applied.sandbox must be one of ${SANDBOXES.join(', ')}`); + if (data.reason !== undefined) { + if (data.sandbox !== 'none') fail('sandbox.applied.reason is set only when sandbox is none'); + if (!isText(data.reason)) fail('sandbox.applied.reason must be a non-empty string'); + } + return data; + }, 'egress.allowed': (data) => { refuseUnknownKeys(data, ['host', 'port'], 'egress.allowed.'); checkEgressTarget(data, 'egress.allowed', { nullable: false }); diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 839679261c..c3b5e8941d 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { - appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, SANDBOXES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -426,3 +426,22 @@ test('egress.denied: {host, port, reason}, the reason from a closed set, null ta assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 70000, reason: 'not_allowed' }), /egress.denied.port/); assert.throws(() => egress('egress.denied', { host: 'evil.example', port: 443, reason: 'not_allowed', status: 403 }), /unknown field egress.denied.status/); }); + +test('sandbox.applied: {sandbox, reason?}, the reason only when the run is unsandboxed', () => { + assert.deepEqual(KINDS['sandbox.applied'], { group: 'sandbox', actor: 'system' }); + assert.deepEqual(SANDBOXES, ['seatbelt', 'bwrap', 'none']); + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'sandbox.applied', data }, { runActor: AGENT })); + const applied = at({ sandbox: 'seatbelt' }); + assert.deepEqual(applied.data, { sandbox: 'seatbelt' }); + assert.deepEqual(applied.actor, { type: 'system', id: 'seats' }); + // sandbox: one of seatbelt, bwrap, none. + at({ sandbox: 'bwrap' }); + for (const sandbox of ['docker', undefined, null, 'Seatbelt']) assert.throws(() => at({ sandbox }), /sandbox.applied.sandbox/, `sandbox ${sandbox}`); + // reason: a non-empty string, and only beside none. + assert.deepEqual(at({ sandbox: 'none', reason: 'no supported OS sandbox on win32' }).data, { sandbox: 'none', reason: 'no supported OS sandbox on win32' }); + at({ sandbox: 'none' }); + assert.throws(() => at({ sandbox: 'seatbelt', reason: 'because' }), /only when sandbox is none/); + for (const reason of ['', 42, null]) assert.throws(() => at({ sandbox: 'none', reason }), /sandbox.applied.reason/, `reason ${reason}`); + // No other field. + assert.throws(() => at({ sandbox: 'bwrap', egress: 'proxy' }), /unknown field sandbox.applied.egress/); +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index 8553f026e8..7493a373be 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -207,3 +207,20 @@ test('egress entries are events on the span current at the time', () => { assert.equal(attrOf(denied, 'seats.reason').stringValue, 'not_allowed'); assert.equal(attrOf(attempt, 'error.type'), undefined, 'a denial is an event, not an error status'); }); + +test('sandbox.applied is an event on the span current at the time', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-sbx-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'sandbox.applied', data: { sandbox: 'none', reason: 'no supported OS sandbox on win32' } }, + { ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'attempt.started', actor: { type: 'agent', id: 'u-agent-ana' }, data: { attempt: 1 } }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'sandbox.applied', data: { sandbox: 'seatbelt' } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const none = spans.find((s) => s.name === 'seats.run').events.find((e) => e.name === 'sandbox.applied'); + assert.equal(attrOf(none, 'seats.sandbox').stringValue, 'none'); + assert.equal(attrOf(none, 'seats.reason').stringValue, 'no supported OS sandbox on win32'); + const seatbelt = spans.find((s) => s.name.startsWith('invoke_agent')).events.find((e) => e.name === 'sandbox.applied'); + assert.equal(attrOf(seatbelt, 'seats.sandbox').stringValue, 'seatbelt'); + assert.equal(attrOf(seatbelt, 'seats.reason'), undefined); +}); diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js index 8e133aaa10..c259330d75 100644 --- a/packages/seats/test/record-store.test.js +++ b/packages/seats/test/record-store.test.js @@ -173,3 +173,12 @@ test('the store records the egress proxy\'s trace types', () => withStore((store assert.deepEqual(store.readRun('run-st-egr1').map((e) => e.kind), ['egress.allowed', 'egress.denied', 'egress.denied', 'run.settled']); assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-egr1'), 'utf8')).errors, []); })); + +test('the store records sandbox.applied', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-sbx1', actor: AGENT }); + run.emit('sandbox.applied', { sandbox: 'none', reason: 'no supported OS sandbox on win32' }); + assert.throws(() => run.emit('sandbox.applied', { sandbox: 'bwrap', reason: 'x' }), /only when sandbox is none/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-sbx1').map((e) => e.kind), ['sandbox.applied', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-sbx1'), 'utf8')).errors, []); +})); From e3f62640cbc99267dd8dc16d3093ec822efa3171 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:34:14 -0700 Subject: [PATCH 11/20] feat(seats): record session.opened, and context fill on usage The ACP adapter (5db73fa3d) emits session.opened {method, sessionId, requestedSessionId?, fallback?} and adds contextTokens and contextSize to usage. session.opened is a v1 kind in the attempt group, default actor system: method is SESSION_METHODS (new, resume, load); sessionId is required; requestedSessionId and fallback come together and only on a new session. fallback is not a closed set in the adapter, since it carries the JSON-RPC error code, so it is pinned to the adapter's two forms: "agent supports neither session/resume nor session/load" and "session/<resume|load> answered error <integer>". usage's two new fields are integers >= 0 or null. OTLP: session.opened is an event on the current span, and its sessionId becomes the attempt's gen_ai.conversation.id when attempt.started had none. The attempt span gains seats.usage.context_tokens and seats.usage.context_size, the last known values in the attempt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 28 ++++++++++++--- packages/seats/src/record/format.js | 28 +++++++++++++++ packages/seats/src/record/otel.js | 19 ++++++++-- packages/seats/test/record-format.test.js | 43 ++++++++++++++++++++++- packages/seats/test/record-otel.test.js | 32 +++++++++++++++++ packages/seats/test/record-store.test.js | 11 ++++++ 6 files changed, 153 insertions(+), 8 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 97c21a0e21..20c003fe43 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -243,6 +243,7 @@ request, and the OTLP export places it there. | kind | data | default actor | |---|---|---| | `attempt.started` | `{attempt, adapter, cli?: {name, version}, sessionId?}` | run | +| `session.opened` | `{method, sessionId, requestedSessionId?, fallback?}` | `system` | | `attempt.settled` | `{attempt, outcome, failure?: {kind, detail}}` | `system` | | `run.settled` | `{outcome, failure?, durationMs, attempts}` | `system` | @@ -250,6 +251,16 @@ request, and the OTLP export places it there. - `failure.kind` is a key of `FAILURE_KINDS` in `src/failures.js`, or `null` when the cause had no name. - The agent's id and owner are the actor of `attempt.started`. +- `session.opened` records the agent session an attempt runs in, as an ACP + adapter opens it: + - `method` is `new`, `resume` or `load`; + - `sessionId` is the session the attempt runs in, a non-empty string; + - `requestedSessionId` and `fallback` are set together, only when + `method` is `new`: a session was asked for and could not be reopened. + `fallback` is one of two forms, `agent supports neither session/resume + nor session/load`, or `session/<resume|load> answered error <code>` + with the JSON-RPC error's integer code; + - no other field is allowed. - `run.settled` is the last line of a complete record. Nothing is written after it. - If an attempt has no `attempt.settled`, the record says only that the next @@ -386,6 +397,10 @@ documented in `src/trace/events.js`. | `agent.message_chunk`, `agent.thought_chunk`, `plan`, `usage` | run | | `liveness.stalled`, `liveness.recovery`, `process.exited` | `system` | +`usage` may also carry `contextTokens` (how much of the agent's context +window is in use) and `contextSize` (the window's size). Each is an integer +`>= 0`, or `null` when unknown; a writer refuses any other value. + ### Mapping from the trace vocabulary Emitters keep calling `emit(type, data)` with `EVENT_TYPES` names, and the @@ -436,9 +451,10 @@ What's new or changed in `EVENT_TYPES`: - The verifier checks the envelope and the chain, never data, so those lines still verify. - A later revision, also at `v: 1`, added the optional `actor.onBehalfOf` - and the kinds `sandbox.applied`, `egress.allowed` and `egress.denied`. - Lines without them are unchanged. + A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`, + the kinds `session.opened`, `sandbox.applied`, `egress.allowed` and + `egress.denied`, and `contextTokens` and `contextSize` on `usage`. Lines + without them are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't @@ -491,7 +507,7 @@ operates it. the time. - **Every other kind is an event on the span current at the time.** That covers grants, credentials, claims, `task.done`, results, liveness, - `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. + `session.opened`, `sandbox.applied`, egress and `tool.update`. An egress denial is an event, not an error status. - **Span ids** are the first 8 bytes of SHA-256(`"seats.record/span/" + runId + "/" + seq`), where `seq` belongs to the entry that opens the span. The run span uses `0`. Exporting the same @@ -527,7 +543,7 @@ to that commit. `error.type` is Stable. | `gen_ai.operation.name` | attempt, tool | `invoke_agent`, `execute_tool` | | `gen_ai.agent.id` | attempt | `actor.id` of `attempt.started` | | `gen_ai.agent.name` | attempt, tool | `actor.name`, else `actor.id` | -| `gen_ai.conversation.id` | attempt | `sessionId` | +| `gen_ai.conversation.id` | attempt | `sessionId` of `attempt.started`, else of the attempt's `session.opened` | | `gen_ai.tool.name` | tool | `name`, else `title` | | `gen_ai.tool.call.id` | tool | `toolCallId` | | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | attempt | the last `usage` entry in the attempt | @@ -545,6 +561,8 @@ to that commit. `error.type` is Stable. - these named attributes: `seats.run.id`, `seats.task.id`, `seats.pod.id`, `seats.run.outcome`, `seats.run.attempts`, `seats.attempt.number`, `seats.attempt.outcome`, `seats.adapter`, `seats.cli.name`, + `seats.usage.context_tokens`, `seats.usage.context_size` (the last + known values in the attempt), `seats.cli.version`, `seats.agent.owner`, `seats.agent.on_behalf_of.id`, `seats.agent.on_behalf_of.role`, `seats.tool.kind`, `seats.tool.surface` and `seats.tool.args_digest`; diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 1e70e095b7..2704af4ddc 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -31,6 +31,10 @@ export const EGRESS_DENIED_REASONS = Object.freeze([ ]); const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); export const SANDBOXES = Object.freeze(['seatbelt', 'bwrap', 'none']); +export const SESSION_METHODS = Object.freeze(['new', 'resume', 'load']); +// Why a requested session was not reopened: the two forms an ACP adapter +// writes. The code is the JSON-RPC error's, an integer by JSON-RPC 2.0. +const SESSION_FALLBACK = /^(?:agent supports neither session\/resume nor session\/load|session\/(?:resume|load) answered error -?\d{1,16})$/; export const REDACTED = '[redacted]'; /** The actors a writer records for its own bookkeeping, and for the board. */ @@ -52,6 +56,7 @@ export const KINDS = Object.freeze({ 'permission.decided': Object.freeze({ group: 'approval', actor: 'decider' }), 'decision.ruled': Object.freeze({ group: 'approval', actor: 'decider' }), 'attempt.started': Object.freeze({ group: 'attempt', actor: 'run', trace: 'run.started' }), + 'session.opened': Object.freeze({ group: 'attempt', actor: 'system' }), 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), 'tool.call': Object.freeze({ group: 'tool', actor: 'run' }), @@ -352,6 +357,29 @@ const CHECKS = { } return data; }, + 'session.opened': (data) => { + refuseUnknownKeys(data, ['method', 'sessionId', 'requestedSessionId', 'fallback'], 'session.opened.'); + if (!SESSION_METHODS.includes(data.method)) fail(`session.opened.method must be one of ${SESSION_METHODS.join(', ')}`); + if (!isText(data.sessionId)) fail('session.opened needs a sessionId'); + if ((data.requestedSessionId === undefined) !== (data.fallback === undefined)) { + fail('session.opened.requestedSessionId and fallback come together: a session was asked for and not reopened'); + } + if (data.requestedSessionId !== undefined) { + if (data.method !== 'new') fail('session.opened.requestedSessionId is set only when method is new'); + if (!isText(data.requestedSessionId)) fail('session.opened.requestedSessionId must be a non-empty string'); + if (typeof data.fallback !== 'string' || !SESSION_FALLBACK.test(data.fallback)) { + fail('session.opened.fallback must be "agent supports neither session/resume nor session/load" or "session/<resume|load> answered error <code>"'); + } + } + return data; + }, + usage: (data) => { + for (const key of ['contextTokens', 'contextSize']) { + const value = data[key]; + if (value !== undefined && value !== null && !(Number.isInteger(value) && value >= 0)) fail(`usage.${key} must be an integer >= 0, or null when unknown`); + } + return data; + }, 'sandbox.applied': (data) => { refuseUnknownKeys(data, ['sandbox', 'reason'], 'sandbox.applied.'); if (!SANDBOXES.includes(data.sandbox)) fail(`sandbox.applied.sandbox must be one of ${SANDBOXES.join(', ')}`); diff --git a/packages/seats/src/record/otel.js b/packages/seats/src/record/otel.js index cf97eff523..6d59ff26f7 100644 --- a/packages/seats/src/record/otel.js +++ b/packages/seats/src/record/otel.js @@ -106,7 +106,7 @@ const spansOfRun = (entries) => { attributes: attrs([['seats.run.id', head.runId], ['seats.task.id', head.taskId], ['seats.pod.id', head.podId]]), }); const spans = [run]; - let attempt = null; // { span, number, tools: Map<toolCallId, span>, usage } + let attempt = null; // { span, number, tools: Map<toolCallId, span>, usage, context, sessionId } const cards = new Map(); // decisionCardId → the span its request sits on const end = (s, ts) => { if (s && s.endTimeUnixNano === null) s.endTimeUnixNano = nanos(ts); }; const closeAttempt = (ts) => { @@ -117,6 +117,13 @@ const spansOfRun = (entries) => { ['gen_ai.usage.input_tokens', attempt.usage.inputTokens], ['gen_ai.usage.output_tokens', attempt.usage.outputTokens], ])); } + attempt.span.attributes.push(...attrs([ + ['seats.usage.context_tokens', attempt.context.tokens], ['seats.usage.context_size', attempt.context.size], + ])); + // The session an attempt opened names the conversation when attempt.started did not. + if (attempt.sessionId && !attempt.span.attributes.some((a) => a.key === 'gen_ai.conversation.id')) { + attempt.span.attributes.push(attr('gen_ai.conversation.id', attempt.sessionId)); + } end(attempt.span, ts); attempt = null; }; @@ -144,7 +151,9 @@ const spansOfRun = (entries) => { ]), }); spans.push(s); - attempt = { span: s, number: data.attempt, agentName, tools: new Map(), usage: null }; + attempt = { + span: s, number: data.attempt, agentName, tools: new Map(), usage: null, context: { tokens: null, size: null }, sessionId: null, + }; } else if (kind === 'attempt.settled' && attempt && attempt.number === data.attempt) { attempt.span.attributes.push(attr('seats.attempt.outcome', data.outcome)); if (FAILED_OUTCOMES.has(data.outcome)) fail(attempt.span, data.failure?.kind || data.outcome); @@ -183,11 +192,17 @@ const spansOfRun = (entries) => { if (tool && (data.status === 'completed' || data.status === 'failed')) end(tool, entry.ts); } else if (kind === 'usage' && attempt) { attempt.usage = data; + // Context fill comes from the stream's usage updates; a later entry without it keeps the last known. + if (Number.isInteger(data.contextTokens)) attempt.context.tokens = data.contextTokens; + if (Number.isInteger(data.contextSize)) attempt.context.size = data.contextSize; attempt.span.events.push(eventOf(entry)); } else if (kind === 'permission.requested' || kind === 'permission.decided') { const target = attempt?.tools.get(data.toolCallId) || current(); target.events.push(eventOf(entry)); if (data.decisionCardId && !cards.has(data.decisionCardId)) cards.set(data.decisionCardId, target); + } else if (kind === 'session.opened') { + if (attempt && typeof data.sessionId === 'string') attempt.sessionId = data.sessionId; + current().events.push(eventOf(entry)); } else if (kind === 'decision.ruled') { // A ruling on a card that a request rode on joins that request; any other is an event where it happened. (cards.get(data.decisionCardId) || current()).events.push(eventOf(entry)); diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index c3b5e8941d..2880ec3986 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -2,7 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { - appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, SANDBOXES, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, SANDBOXES, SESSION_METHODS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -445,3 +445,44 @@ test('sandbox.applied: {sandbox, reason?}, the reason only when the run is unsan // No other field. assert.throws(() => at({ sandbox: 'bwrap', egress: 'proxy' }), /unknown field sandbox.applied.egress/); }); + +test('session.opened: {method, sessionId, requestedSessionId?, fallback?}, the pair only when a new session replaced a requested one', () => { + assert.deepEqual(KINDS['session.opened'], { group: 'attempt', actor: 'system' }); + assert.deepEqual(SESSION_METHODS, ['new', 'resume', 'load']); + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'session.opened', data }, { runActor: AGENT })); + const opened = at({ method: 'resume', sessionId: 'ses-1' }); + assert.deepEqual(opened.data, { method: 'resume', sessionId: 'ses-1' }); + assert.deepEqual(opened.actor, { type: 'system', id: 'seats' }); + // method: new, resume or load. + at({ method: 'load', sessionId: 'ses-1' }); + for (const method of ['open', undefined, 'NEW']) assert.throws(() => at({ method, sessionId: 'ses-1' }), /session.opened.method/, `method ${method}`); + // sessionId: required, a non-empty string. + at({ method: 'new', sessionId: '01JB5Z' }); + for (const sessionId of [undefined, '', 7]) assert.throws(() => at({ method: 'new', sessionId }), /needs a sessionId/, `sessionId ${sessionId}`); + // requestedSessionId and fallback: together, and only on a new session. + const replaced = { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1' }; + assert.deepEqual(at({ ...replaced, fallback: 'session/resume answered error -32603' }).data, { ...replaced, fallback: 'session/resume answered error -32603' }); + at({ ...replaced, fallback: 'agent supports neither session/resume nor session/load' }); + at({ ...replaced, fallback: 'session/load answered error 404' }); + assert.throws(() => at(replaced), /come together/); + assert.throws(() => at({ method: 'new', sessionId: 'ses-2', fallback: 'session/load answered error 404' }), /come together/); + assert.throws(() => at({ ...replaced, method: 'resume', fallback: 'session/load answered error 404' }), /only when method is new/); + assert.throws(() => at({ ...replaced, requestedSessionId: '', fallback: 'session/load answered error 404' }), /requestedSessionId must be/); + // fallback: one of the two forms the adapter writes, nothing else. + for (const fallback of ['it broke', 'session/new answered error 1', 'session/load answered error x', 'session/load answered error 1; rm -rf', null, '']) { + assert.throws(() => at({ ...replaced, fallback }), /session.opened.fallback/, `fallback ${JSON.stringify(fallback)}`); + } + assert.throws(() => at({ method: 'new', sessionId: 'ses-2', cwd: '/home/ana' }), /unknown field session.opened.cwd/); +}); + +test('usage: contextTokens and contextSize are integers >= 0, or null when unknown', () => { + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'usage', data }, { runActor: AGENT })); + const filled = at({ inputTokens: null, outputTokens: null, totalTokens: null, cost: null, contextTokens: 41000, contextSize: 200000 }); + assert.equal(filled.data.contextTokens, 41000); + assert.equal(filled.data.contextSize, 200000); + at({ contextTokens: 0, contextSize: null }); + at({ inputTokens: 10, outputTokens: 2 }); + for (const key of ['contextTokens', 'contextSize']) { + for (const value of [-1, 1.5, '41000', Number.NaN]) assert.throws(() => at({ [key]: value }), new RegExp(`usage.${key}`), `${key} ${value}`); + } +}); diff --git a/packages/seats/test/record-otel.test.js b/packages/seats/test/record-otel.test.js index 7493a373be..b1aa6fd44d 100644 --- a/packages/seats/test/record-otel.test.js +++ b/packages/seats/test/record-otel.test.js @@ -224,3 +224,35 @@ test('sandbox.applied is an event on the span current at the time', () => { assert.equal(attrOf(seatbelt, 'seats.sandbox').stringValue, 'seatbelt'); assert.equal(attrOf(seatbelt, 'seats.reason'), undefined); }); + +test('session.opened is an event that names the attempt\'s conversation; context usage lands on the attempt span', () => { + const base = { v: 1, schema: 'seats.record', runId: 'run-ses-0001', prev: null, actor: { type: 'system', id: 'seats' } }; + const agent = { type: 'agent', id: 'u-agent-ana' }; + const entries = [ + { ...base, seq: 1, ts: '2026-10-25T12:00:00.000Z', kind: 'attempt.started', actor: agent, data: { attempt: 1, adapter: 'acp' } }, + { + ...base, seq: 2, ts: '2026-10-25T12:00:01.000Z', kind: 'session.opened', + data: { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1', fallback: 'session/resume answered error -32601' }, + }, + { ...base, seq: 3, ts: '2026-10-25T12:00:02.000Z', kind: 'usage', actor: agent, data: { contextTokens: 900, contextSize: 128000 } }, + { ...base, seq: 4, ts: '2026-10-25T12:00:03.000Z', kind: 'usage', actor: agent, data: { inputTokens: 1200, outputTokens: 80, contextTokens: null } }, + { ...base, seq: 5, ts: '2026-10-25T12:00:04.000Z', kind: 'attempt.started', actor: agent, data: { attempt: 2, adapter: 'acp', sessionId: 'ses-9' } }, + { ...base, seq: 6, ts: '2026-10-25T12:00:05.000Z', kind: 'session.opened', data: { method: 'resume', sessionId: 'ses-other' } }, + { ...base, seq: 7, ts: '2026-10-25T12:00:06.000Z', kind: 'run.settled', data: { outcome: 'ok' } }, + ]; + const spans = spansOf(toOtlpJson(entries)); + const [first, second] = spans.filter((s) => s.name.startsWith('invoke_agent')); + const opened = first.events.find((e) => e.name === 'session.opened'); + assert.equal(attrOf(opened, 'seats.method').stringValue, 'new'); + assert.equal(attrOf(opened, 'seats.requested_session_id').stringValue, 'ses-1'); + assert.equal(attrOf(opened, 'seats.fallback').stringValue, 'session/resume answered error -32601'); + assert.equal(attrOf(first, 'gen_ai.conversation.id').stringValue, 'ses-2'); + assert.equal(attrOf(first, 'seats.usage.context_tokens').intValue, '900', 'a later usage without context keeps the last known'); + assert.equal(attrOf(first, 'seats.usage.context_size').intValue, '128000'); + assert.equal(attrOf(first, 'gen_ai.usage.input_tokens').intValue, '1200'); + const seen = first.events.find((e) => e.name === 'usage'); + assert.equal(attrOf(seen, 'seats.context_tokens').intValue, '900'); + // attempt.started's own sessionId wins; one conversation id per span. + assert.deepEqual(second.attributes.filter((a) => a.key === 'gen_ai.conversation.id').map((a) => a.value.stringValue), ['ses-9']); + assert.equal(attrOf(second, 'seats.usage.context_tokens'), undefined); +}); diff --git a/packages/seats/test/record-store.test.js b/packages/seats/test/record-store.test.js index c259330d75..a1b3fd2dc7 100644 --- a/packages/seats/test/record-store.test.js +++ b/packages/seats/test/record-store.test.js @@ -182,3 +182,14 @@ test('the store records sandbox.applied', () => withStore((store, root) => { assert.deepEqual(store.readRun('run-st-sbx1').map((e) => e.kind), ['sandbox.applied', 'run.settled']); assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-sbx1'), 'utf8')).errors, []); })); + +test('the store records session.opened and context usage', () => withStore((store, root) => { + const run = store.startRun({ runId: 'run-st-ses1', actor: AGENT }); + run.emit('run.started', { adapter: 'acp', attempt: 1 }); + run.emit('session.opened', { method: 'new', sessionId: 'ses-2', requestedSessionId: 'ses-1', fallback: 'session/resume answered error -32601' }); + run.emit('usage', { inputTokens: null, outputTokens: null, totalTokens: null, cost: null, contextTokens: 900, contextSize: 128000 }); + assert.throws(() => run.emit('session.opened', { method: 'new', sessionId: 'ses-3', requestedSessionId: 'ses-1' }), /come together/); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assert.deepEqual(store.readRun('run-st-ses1').map((e) => e.kind), ['attempt.started', 'session.opened', 'usage', 'run.settled']); + assert.deepEqual(verifyRecord(readFileSync(fileOf(root, 'run-st-ses1'), 'utf8')).errors, []); +})); From e1bd2cc99569ee61809029ceddde1b40674df77f Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:42:29 -0700 Subject: [PATCH 12/20] test(seats): plant the login home in valet-env instead of reading the host's The positive control required a real ~/.claude and ~/.codex on the machine running the suite, so it failed on a CI runner or a bare container, where neither exists. The test now plants both dirs in a scratch home. It proves a child given that HOME directly can see them, then asserts the valet's run cannot. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/test/valet-env.test.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/seats/test/valet-env.test.js b/packages/seats/test/valet-env.test.js index 5a35adf191..6218f3a5a2 100644 --- a/packages/seats/test/valet-env.test.js +++ b/packages/seats/test/valet-env.test.js @@ -1,7 +1,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readFileSync, readdirSync as readdir, rmSync, statSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync as readdir, rmSync, statSync, writeFileSync } from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { prepareRunEnv } from '../src/valet/env.js'; @@ -34,8 +34,19 @@ const probe = (env, cwd) => { }; test('a real child sees only the granted key, a throwaway HOME, and no path to the real login', () => withScratch(async (root) => { + // A planted "real" home with both login dirs, so the result never depends on the host + // (a CI runner or a bare container has neither). + const loginHome = mkdtempSync(join(tmpdir(), 'seats-login-home-')); + for (const dir of ['.claude', '.codex']) { + mkdirSync(join(loginHome, dir), { mode: 0o700 }); + writeFileSync(join(loginHome, dir, 'credentials.json'), '{"planted":true}'); + } + // Positive control: a child given that HOME directly can reach both, so "not reachable" below is a real result. + const direct = probe({ ...process.env, HOME: loginHome }, loginHome); + assert.equal(direct.claude, true, 'positive control: the probe sees ~/.claude when HOME points at it'); + assert.equal(direct.codex, true, 'positive control: the probe sees ~/.codex when HOME points at it'); // The parent's own env, with both secrets planted, is the base the run is cut from. - const baseEnv = { ...process.env, HOME: REAL_HOME, OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD, CLAUDE_CONFIG_DIR: join(REAL_HOME, '.claude') }; + const baseEnv = { ...process.env, HOME: loginHome, OPENAI_API_KEY: GRANTED, ANTHROPIC_API_KEY: WITHHELD, CLAUDE_CONFIG_DIR: join(loginHome, '.claude') }; const trace = recorder(); const run = await prepareRunEnv({ grant: { credentials: [{ name: 'OPENAI_API_KEY', from: 'env' }] }, baseEnv, root, trace, @@ -50,13 +61,11 @@ test('a real child sees only the granted key, a throwaway HOME, and no path to t assert.equal(seen.homedir, run.home); assert.ok(run.home.startsWith(root)); for (const [name, value] of Object.entries(seen.env)) { + assert.ok(!value.includes(loginHome), `${name} leaks the login HOME path: ${value}`); assert.ok(!value.includes(REAL_HOME), `${name} leaks the real HOME path: ${value}`); } - // Positive control: the real login dir exists, so "not reachable" is a real result. - if (existsSync(join(REAL_HOME, '.claude'))) assert.equal(seen.claude, false, '~/.claude is not reachable via HOME'); - else assert.fail('positive control missing: no ~/.claude on this host'); - if (existsSync(join(REAL_HOME, '.codex'))) assert.equal(seen.codex, false, '~/.codex is not reachable via HOME'); - else assert.fail('positive control missing: no ~/.codex on this host'); + assert.equal(seen.claude, false, '~/.claude is not reachable via HOME'); + assert.equal(seen.codex, false, '~/.codex is not reachable via HOME'); for (const name of ['XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_STATE_HOME', 'XDG_CACHE_HOME', 'TMPDIR']) { assert.ok(seen.env[name].startsWith(`${run.home}/`), `${name} sits inside the run HOME`); assert.equal(statSync(seen.env[name]).mode & 0o777, 0o700, `${name} is 0700`); @@ -66,6 +75,7 @@ test('a real child sees only the granted key, a throwaway HOME, and no path to t assert.deepEqual(trace.events, [{ type: 'credential.granted', data: { names: ['OPENAI_API_KEY'] } }]); } finally { await run.cleanup(); + rmSync(loginHome, { recursive: true, force: true }); } assert.equal(existsSync(run.home), false, 'cleanup removes the HOME'); })); From 79e580dea75a4a2c493cbfb9c4dc48e6766093f6 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:43:55 -0700 Subject: [PATCH 13/20] feat(seats): add pod-member to decidedByRole, valid only on decision.ruled pod-member is a person in the pod who owns neither the credential nor the asking agent. They may rule a decision card that parks no call. A parked call stays owner-only (ADR-033 D5), so permission.decided and permission.requested.askedOf refuse the role. When an approver answers as a pod member, the valet treats the answer as no decision at all. The call stays parked, no permission.decided is written, and approve is called again with `refused: {by, role, reason: 'not_owner'}` so the card can show why and wait for the owner. That repeats within the same timeout. An approver that does nothing but answer as a pod member fails the ask after MAX_REFUSED_ANSWERS, the same as any other broken approver. Adding an enum value keeps the record at v: 1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 11 ++++-- packages/seats/src/record/format.js | 15 +++++---- packages/seats/src/valet/permissions.js | 32 ++++++++++++++---- packages/seats/test/record-format.test.js | 6 ++++ packages/seats/test/record-valet.test.js | 41 ++++++++++++++++++++++- 5 files changed, 89 insertions(+), 16 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 20c003fe43..ce04c3b7a5 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -223,6 +223,10 @@ it does not prove the process lost access. That happens at `run.settled`. - `decision-owner`: a person assigned to the decision. This is reserved, and writers may not emit it yet; - `agent-owner`: the owner of the asking agent; + - `pod-member`: a person in the pod who owns neither the credential nor + the asking agent. It is valid only on `decision.ruled`. A parked call is + decided by an owner or by policy, so `permission.decided` and + `permission.requested.askedOf` refuse it; - `policy`; - `null`, when unknown. - `decisionCardId` is set when the decision came through a decision card. @@ -233,7 +237,8 @@ a parked tool call, such as whether to retry a stuck attempt. - `decisionCardId` is required. - `value` is the ruling: a string, number or boolean. - `decidedBy` is required. It is a user id, or `"policy"`. -- `decidedByRole` is required, and comes from the same enum. +- `decidedByRole` is required, and comes from the same enum. Only here may it + be `pod-member`. A ruling on a card that a `permission.requested` carried belongs with that request, and the OTLP export places it there. @@ -453,8 +458,8 @@ What's new or changed in `EVENT_TYPES`: A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`, the kinds `session.opened`, `sandbox.applied`, `egress.allowed` and - `egress.denied`, and `contextTokens` and `contextSize` on `usage`. Lines - without them are unchanged. + `egress.denied`, `contextTokens` and `contextSize` on `usage`, and the + `pod-member` value of `decidedByRole`. Lines without them are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 2704af4ddc..a2e89036a5 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -14,7 +14,10 @@ export const RECORD_SCHEMA = 'seats.record'; export const ACTOR_TYPES = Object.freeze(['agent', 'human', 'policy', 'system']); // Who decides an approval today is the owner of the credential it would use; // `decision-owner` is reserved for a separately assigned decider. -export const DECIDED_BY_ROLES = Object.freeze(['credential-owner', 'decision-owner', 'agent-owner', 'policy']); +export const DECIDED_BY_ROLES = Object.freeze(['credential-owner', 'decision-owner', 'agent-owner', 'pod-member', 'policy']); +// A parked tool call is decided by an owner or by policy, never by a pod member +// who owns neither the credential nor the asking agent (ADR-033 D5). +export const PARKED_CALL_ROLES = Object.freeze(DECIDED_BY_ROLES.filter((role) => role !== 'pod-member')); export const TOOL_SURFACES = Object.freeze(['connector', 'commonly', 'native']); export const GRANT_ENDS_ON = Object.freeze(['task.done', 'expiresAt']); export const REVOKE_REASONS = Object.freeze(['task.done', 'expired', 'cascade', 'manual']); @@ -220,8 +223,8 @@ const checkEgressTarget = (data, kind, { nullable }) => { if (!isPort(data.port)) fail(`${kind}.port must be an integer from 1 to 65535`); }; -const checkRole = (role, where) => { - if (!DECIDED_BY_ROLES.includes(role)) fail(`${where} must be one of ${DECIDED_BY_ROLES.join(', ')}`); +const checkRole = (role, where, roles = DECIDED_BY_ROLES) => { + if (!roles.includes(role)) fail(`${where} must be one of ${roles.join(', ')}`); }; /** A grant's scope, in the shape of a room grant: exact tool names and a connection id. */ @@ -304,7 +307,7 @@ const CHECKS = { if (!isPlainObject(data.askedOf)) fail('permission.requested needs askedOf: {id, role}'); refuseUnknownKeys(data.askedOf, ['id', 'role'], 'permission.requested.askedOf.'); if (!isUserId(data.askedOf.id)) fail('permission.requested.askedOf.id must be an id token, never a display name'); - checkRole(data.askedOf.role, 'permission.requested.askedOf.role'); + checkRole(data.askedOf.role, 'permission.requested.askedOf.role', PARKED_CALL_ROLES); optionalText(data.grantId, 'permission.requested.grantId'); optionalText(data.decisionCardId, 'permission.requested.decisionCardId'); optionalArgsDigest(data, 'permission.requested'); @@ -332,8 +335,8 @@ const CHECKS = { } if (!('decidedByRole' in out)) out.decidedByRole = null; if (out.decidedBy !== null) userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); - if (out.decidedByRole !== null && !DECIDED_BY_ROLES.includes(out.decidedByRole)) { - fail(`permission.decided.decidedByRole must be one of ${DECIDED_BY_ROLES.join(', ')}, or null`); + if (out.decidedByRole !== null && !PARKED_CALL_ROLES.includes(out.decidedByRole)) { + fail(`permission.decided.decidedByRole must be one of ${PARKED_CALL_ROLES.join(', ')}, or null`); } optionalText(out.decisionCardId, 'permission.decided.decisionCardId'); return out; diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js index 21a08992d1..224c18bf16 100644 --- a/packages/seats/src/valet/permissions.js +++ b/packages/seats/src/valet/permissions.js @@ -9,16 +9,23 @@ * { approved: true, by: <user id>, role?, decisionCardId? }. * Anything else (false, 'yes', {approved: 1}, a throw, no answer in * time) denies. `by`/`role`/`decisionCardId` are recorded either way. + * An answer whose role is `pod-member` decides nothing (ADR-033 D5): + * the call stays parked, and approve is called again with + * `refused: { by, role: 'pod-member', reason: 'not_owner' }` so the + * card can show why and wait for the owner. * * allow_always and reject_always are never chosen: either would be remembered * by the agent past this grant, so the next call would skip the check. * An allow rule with no allow_once option therefore denies. */ import { traceOf } from '../trace/events.js'; -import { argsDigestOf, DECIDED_BY_ROLES, isUserId, policyActorId } from '../record/format.js'; +import { argsDigestOf, PARKED_CALL_ROLES, isUserId, policyActorId } from '../record/format.js'; import { matchToolCall, validateGrant } from './grant.js'; export const DEFAULT_APPROVE_TIMEOUT_MS = 5 * 60 * 1000; +// An approver that keeps handing back pod-member answers is broken; after this +// many the ask fails, and the policy's default decides, as for any broken approver. +export const MAX_REFUSED_ANSWERS = 10; const optionOfKind = (options, kind) => (Array.isArray(options) ? options : []) .find((option) => option && option.kind === kind && typeof option.optionId === 'string'); @@ -46,11 +53,13 @@ const readAnswer = (answer) => { return { verdict: answer.approved === true ? 'approved' : 'rejected', ...(isUserId(answer.by) ? { by: answer.by } : {}), - ...(DECIDED_BY_ROLES.includes(answer.role) ? { role: answer.role } : {}), + ...(PARKED_CALL_ROLES.includes(answer.role) ? { role: answer.role } : {}), ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), }; }; +const isPodMemberAnswer = (answer) => isPlainObject(answer) && answer.role === 'pod-member'; + /** * Wait for the host's approval, bounded. Resolves { verdict, by?, role?, * decisionCardId? }, verdict 'approved' | 'rejected' | 'timed_out' | 'failed' @@ -71,9 +80,20 @@ const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((do const onAbort = () => finish({ verdict: 'aborted' }); const timer = setTimeout(() => finish({ verdict: 'timed_out' }), timeoutMs); signal?.addEventListener?.('abort', onAbort, { once: true }); - Promise.resolve() - .then(() => approve(input)) - .then((answer) => finish(readAnswer(answer)), () => finish({ verdict: 'failed' })); + let refusals = 0; + const ask = (current) => Promise.resolve() + .then(() => approve(current)) + .then((answer) => { + if (finished) return; + if (!isPodMemberAnswer(answer)) { finish(readAnswer(answer)); return; } + // A pod member may not decide a parked call: the call stays parked and + // the owner is asked again, with the refusal, inside the same timeout. + refusals += 1; + if (refusals > MAX_REFUSED_ANSWERS) { finish({ verdict: 'failed' }); return; } + const refused = { by: isUserId(answer.by) ? answer.by : null, role: 'pod-member', reason: 'not_owner' }; + setTimeout(() => { if (!finished) ask({ ...input, refused }); }, 0); + }, () => finish({ verdict: 'failed' })); + ask(input); }); /** @@ -94,7 +114,7 @@ const decider = (approval) => { return { decidedBy: 'policy', decidedByRole: 'policy' }; }; -const isAsker = (value) => isPlainObject(value) && isUserId(value.id) && DECIDED_BY_ROLES.includes(value.role); +const isAsker = (value) => isPlainObject(value) && isUserId(value.id) && PARKED_CALL_ROLES.includes(value.role); /** * Who an ask goes to, for permission.requested. `askedOf` is {id, role} or diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 2880ec3986..3e5b955065 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -220,6 +220,8 @@ test('permission.decided: decidedBy is separate from the agent owner, and creden assert.equal(asked.data.decidedByRole, 'credential-owner'); for (const role of ['decision-owner', 'agent-owner', 'policy']) decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: role }); assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'owner' }), /decidedByRole/); + // A parked call is decided by an owner or by policy, never a pod member (ADR-033 D5). + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-cal', decidedByRole: 'pod-member' }), /decidedByRole/); assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'Ben Smith' }), /decidedBy/); const byRule = decide({ outcome: 'selected', optionId: 'reject', kind: 'reject_once', toolCallId: 't', rule: { list: 'deny', pattern: 'x' }, grantId: 'g-1' }); assert.equal(byRule.data.decidedBy, 'policy'); @@ -271,6 +273,7 @@ test('permission.requested: question and askedOf are required, askedOf.id is an assert.throws(() => request(askData({ askedOf: { id: 'Ben Smith', role: 'credential-owner' } })), /askedOf.id/); assert.throws(() => request(askData({ askedOf: { id: 'u-ben' } })), /askedOf.role/); assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'boss' } })), /askedOf.role/); + assert.throws(() => request(askData({ askedOf: { id: 'u-cal', role: 'pod-member' } })), /askedOf.role/, 'a parked call is not asked of a pod member'); assert.throws(() => request(askData({ askedOf: { id: 'u-ben', role: 'policy', email: 'ben@example.com' } })), /unknown field/); assert.throws(() => request(askData({ decisionCardId: '' })), /decisionCardId/); assert.equal(request(askData({ askedOf: { id: 'grant:g-1', role: 'policy' } })).data.askedOf.id, 'grant:g-1'); @@ -284,6 +287,9 @@ test('decision.ruled: a card ruling, validated like permission.decided', () => { assert.deepEqual(entry.actor, { type: 'human', id: 'u-ana' }, 'the ruler is the actor'); assert.deepEqual(rule({ ...good, decidedBy: 'policy', decidedByRole: 'policy' }).actor, { type: 'policy', id: 'policy' }); for (const value of [true, 3, 'ship']) rule({ ...good, value }); + const byMember = rule({ ...good, decidedBy: 'u-cal', decidedByRole: 'pod-member' }); + assert.equal(byMember.data.decidedByRole, 'pod-member', 'a pod member may rule a card that parks no call'); + assert.deepEqual(byMember.actor, { type: 'human', id: 'u-cal' }); assert.throws(() => rule({ ...good, decisionCardId: undefined }), /decisionCardId/); assert.throws(() => rule({ ...good, decidedBy: 'Ana Smith' }), /decidedBy/); assert.throws(() => rule({ ...good, decidedBy: undefined }), /decidedBy/); diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js index 905c43dbeb..c88ad82324 100644 --- a/packages/seats/test/record-valet.test.js +++ b/packages/seats/test/record-valet.test.js @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createTraceStore } from '../src/trace/store.js'; import { prepareRunEnv } from '../src/valet/env.js'; -import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { createPermissionPolicy, MAX_REFUSED_ANSWERS } from '../src/valet/permissions.js'; import { argsDigestOf } from '../src/record/format.js'; const FROM_ENV = 'sk-env-planted-0a1b2c3d4e5f'; @@ -134,6 +134,45 @@ test('the decision names who decided and as what, separate from the agent and it assert.equal(bare.data.decidedBy, null, 'a bare true names nobody'); }); +test('a pod member\'s answer decides nothing: the call stays parked until the owner answers (ADR-033 D5)', async () => { + const events = []; + const seen = []; + let ownerAnswers; + const ownerAnswer = new Promise((resolve) => { ownerAnswers = resolve; }); + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approveTimeoutMs: 5000, grantId: 'g-9', + approve: (input) => { + seen.push(input); + return seen.length === 1 ? { approved: true, by: 'u-cal', role: 'pod-member', decisionCardId: 'card-1' } : ownerAnswer; + }, + }); + const pending = decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + while (seen.length < 2) await new Promise((resolve) => { setTimeout(resolve, 5); }); + assert.deepEqual(events.map((e) => e.type), ['permission.requested'], 'no permission.decided for the pod member\'s answer'); + assert.deepEqual(seen[1].refused, { by: 'u-cal', role: 'pod-member', reason: 'not_owner' }, 'the card is told why'); + assert.equal(seen[0].refused, undefined); + ownerAnswers({ approved: true, by: 'u-ben', role: 'credential-owner', decisionCardId: 'card-1' }); + assert.deepEqual(await pending, { outcome: 'selected', optionId: 'allow' }); + const decided = events.filter((e) => e.type === 'permission.decided'); + assert.equal(decided.length, 1); + assert.equal(decided[0].data.decidedBy, 'u-ben'); + assert.equal(decided[0].data.decidedByRole, 'credential-owner'); +}); + +test('an approver that only ever answers as a pod member fails the ask, and the policy default decides', async () => { + const events = []; + let calls = 0; + const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { + trace: { emit: (type, data) => events.push({ type, data }) }, approveTimeoutMs: 5000, grantId: 'g-9', + approve: () => { calls += 1; return { approved: true, by: 'u-cal', role: 'pod-member' }; }, + }); + const result = await decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }); + assert.equal(calls, MAX_REFUSED_ANSWERS + 1); + const [decided] = events.filter((e) => e.type === 'permission.decided'); + assert.deepEqual([decided.data.decidedBy, decided.data.decidedByRole], ['policy', 'policy']); +}); + test('no answer in time is decided by policy', async () => { const events = []; const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { From ccfd71c4ef32fa412cc77ae7402b19971be94351 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:01:30 -0700 Subject: [PATCH 14/20] feat(seats): a valet policy with a trace marks itself recordsPermissions createPermissionPolicy writes permission.requested and permission.decided itself when it is given a trace. An adapter handed it as ctx.decidePermission reads the non-enumerable recordsPermissions marker and skips its own writes, so each permission is recorded once. This replaces a ctx flag whose default would have written every permission twice. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/src/valet/permissions.js | 7 ++++++- packages/seats/test/record-valet.test.js | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js index 224c18bf16..ecad31093e 100644 --- a/packages/seats/src/valet/permissions.js +++ b/packages/seats/src/valet/permissions.js @@ -148,6 +148,7 @@ const askOf = (askedOf, input, policy) => { * permission.requested: the question (the tool call's title), who it is * asked of, and the card it rides on. The approver receives that card id. * When the tool call carries `rawInput`, both entries carry its argsDigest. + * Given a `trace`, the returned function has `recordsPermissions === true`. */ export const createPermissionPolicy = (grant, { approve, trace, approveTimeoutMs = DEFAULT_APPROVE_TIMEOUT_MS, signal, grantId, askedOf, @@ -158,7 +159,7 @@ export const createPermissionPolicy = (grant, { throw new Error('askedOf must be {id: <user id>, role: <decidedByRole>} or a function returning one'); } const policy = { id: policyActorId(grantId), role: 'policy' }; - return async (request = {}) => { + const decide = async (request = {}) => { const toolCall = request?.toolCall || {}; const options = request?.options; const { decision, rule } = matchToolCall(valid, toolCall); @@ -200,4 +201,8 @@ export const createPermissionPolicy = (grant, { }); return answer.outcome === 'selected' ? { outcome: 'selected', optionId: answer.optionId } : { outcome: 'cancelled' }; }; + // With a trace this writes both permission entries itself, so an adapter + // handed it as ctx.decidePermission must not write them again. + if (trace) Object.defineProperty(decide, 'recordsPermissions', { value: true }); + return decide; }; diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js index c88ad82324..886e87618e 100644 --- a/packages/seats/test/record-valet.test.js +++ b/packages/seats/test/record-valet.test.js @@ -173,6 +173,13 @@ test('an approver that only ever answers as a pod member fails the ask, and the assert.deepEqual([decided.data.decidedBy, decided.data.decidedByRole], ['policy', 'policy']); }); +test('a valet policy with a trace says it records permissions itself; without one it does not', () => { + const recording = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { trace: { emit: () => {} } }); + assert.equal(recording.recordsPermissions, true); + assert.equal(Object.keys(recording).includes('recordsPermissions'), false, 'non-enumerable'); + assert.equal(createPermissionPolicy({ tools: { ask: ['fetch'] } }).recordsPermissions, undefined); +}); + test('no answer in time is decided by policy', async () => { const events = []; const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { From 92d6d096e9b9d53faa0d3c8dbb23af08ae3be5d9 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:08:01 -0700 Subject: [PATCH 15/20] fix(seats): a decision always names who made it; an unattributed answer approves nothing An approver's answer now decides a parked call only when it is an object whose `approved` is exactly true or false and that names a user id and an owner role. A bare `true`, or any answer naming nobody, is a failed ask: the policy's default, deny, decides and is recorded as decidedBy/decidedByRole "policy". The record format refuses a permission.decided line with a null or missing decidedBy or decidedByRole, and requires decidedBy "policy" exactly when the role is "policy", so no future writer can record a decision nobody can be held to. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 14 +++--- packages/seats/src/record/format.js | 18 ++++--- packages/seats/src/valet/permissions.js | 39 ++++++++------- packages/seats/test/record-format.test.js | 17 +++++-- packages/seats/test/record-valet.test.js | 49 ++++++++++++++----- packages/seats/test/valet-permissions.test.js | 12 +++-- 6 files changed, 96 insertions(+), 53 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index ce04c3b7a5..335fd202d8 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -213,10 +213,11 @@ it does not prove the process lost access. That happens at `run.settled`. `rejected`, `timed_out`, `failed` or `aborted`. **Who decided:** -- `decidedBy` is a user id, `"policy"`, or `null`. - - `null` means the approver answered without saying who it was. - - A rule-decided call, or the default given when no answer came in time, is - `"policy"`. +- `decidedBy` is required: a user id, or `"policy"`. A writer refuses `null`. + - A rule-decided call is `"policy"`. So is the default given when no + answer came in time, or when the answer did not name a user and an owner + role: such an answer approves nothing. + - It is `"policy"` exactly when `decidedByRole` is `"policy"`. - `decidedByRole` is one of: - `credential-owner`: the owner of the connection the call would use. This is who decides approvals today; @@ -227,8 +228,9 @@ it does not prove the process lost access. That happens at `run.settled`. the asking agent. It is valid only on `decision.ruled`. A parked call is decided by an owner or by policy, so `permission.decided` and `permission.requested.askedOf` refuse it; - - `policy`; - - `null`, when unknown. + - `policy`. + + It is required on `permission.decided`, and a writer refuses `null`. - `decisionCardId` is set when the decision came through a decision card. - `grantId` ties the decision to its `grant.issued`. diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index a2e89036a5..53027db99c 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -327,16 +327,18 @@ const CHECKS = { optionalArgsDigest(data, 'permission.decided'); // A rule-decided call never had a person in the loop; a writer says so // rather than leave a reader to infer it from rule.list. + // A decision always names who made it: no writer may record one nobody + // can be held to. const out = { ...data }; - if (!('decidedBy' in out)) { - const byRule = out.rule && out.rule.list !== 'ask'; - out.decidedBy = byRule ? 'policy' : null; - if (!('decidedByRole' in out)) out.decidedByRole = byRule ? 'policy' : null; + if (!('decidedBy' in out) && !('decidedByRole' in out) && out.rule && out.rule.list !== 'ask') { + out.decidedBy = 'policy'; + out.decidedByRole = 'policy'; } - if (!('decidedByRole' in out)) out.decidedByRole = null; - if (out.decidedBy !== null) userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); - if (out.decidedByRole !== null && !PARKED_CALL_ROLES.includes(out.decidedByRole)) { - fail(`permission.decided.decidedByRole must be one of ${PARKED_CALL_ROLES.join(', ')}, or null`); + if (out.decidedBy === undefined || out.decidedBy === null) fail('permission.decided.decidedBy is required: a user id, or "policy"'); + userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); + checkRole(out.decidedByRole, 'permission.decided.decidedByRole', PARKED_CALL_ROLES); + if ((out.decidedBy === 'policy') !== (out.decidedByRole === 'policy')) { + fail('permission.decided.decidedBy is "policy" exactly when decidedByRole is "policy"'); } optionalText(out.decisionCardId, 'permission.decided.decisionCardId'); return out; diff --git a/packages/seats/src/valet/permissions.js b/packages/seats/src/valet/permissions.js index ecad31093e..a6445688c1 100644 --- a/packages/seats/src/valet/permissions.js +++ b/packages/seats/src/valet/permissions.js @@ -4,11 +4,12 @@ * * allow → the allow_once option * deny → the reject_once option, or cancelled when there is none - * ask → approve({ toolCall, grantRule }) from the host. `true` allows, - * and so does an object whose `approved` is exactly true: - * { approved: true, by: <user id>, role?, decisionCardId? }. - * Anything else (false, 'yes', {approved: 1}, a throw, no answer in - * time) denies. `by`/`role`/`decisionCardId` are recorded either way. + * ask → approve({ toolCall, grantRule }) from the host. Only an answer + * that says who gave it decides: + * { approved: true | false, by: <user id>, role, decisionCardId? }, + * with `role` an owner role. Anything else (a bare true, 'yes', + * {approved: 1}, an answer naming nobody, a throw, no answer in + * time) is a failed ask, and the policy's default, deny, decides. * An answer whose role is `pod-member` decides nothing (ADR-033 D5): * the call stays parked, and approve is called again with * `refused: { by, role: 'pod-member', reason: 'not_owner' }` so the @@ -42,18 +43,22 @@ const denyAnswer = (options) => { const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); const isText = (value) => typeof value === 'string' && value.length > 0; +// The roles a person can answer a parked call as; `policy` is not a person. +const ANSWER_ROLES = PARKED_CALL_ROLES.filter((role) => role !== 'policy'); + /** - * What an approver's answer says. Only `true`, or an object with - * `approved === true`, approves; attribution is taken from an object either - * way, and a malformed field is dropped rather than trusted. + * What an approver's answer says. It approves or rejects only when it is an + * object whose `approved` is exactly true or false and that names a user id + * and an owner role. Any other answer approves nothing: it is a failed ask, + * so the record never carries a decision nobody can be held to. */ const readAnswer = (answer) => { - if (answer === true) return { verdict: 'approved' }; - if (!isPlainObject(answer)) return { verdict: 'rejected' }; + if (!isPlainObject(answer) || typeof answer.approved !== 'boolean') return { verdict: 'failed' }; + if (!isUserId(answer.by) || !ANSWER_ROLES.includes(answer.role)) return { verdict: 'failed' }; return { - verdict: answer.approved === true ? 'approved' : 'rejected', - ...(isUserId(answer.by) ? { by: answer.by } : {}), - ...(PARKED_CALL_ROLES.includes(answer.role) ? { role: answer.role } : {}), + verdict: answer.approved ? 'approved' : 'rejected', + by: answer.by, + role: answer.role, ...(isText(answer.decisionCardId) ? { decisionCardId: answer.decisionCardId } : {}), }; }; @@ -98,16 +103,16 @@ const awaitApproval = (approve, input, { timeoutMs, signal }) => new Promise((do /** * Who decided an ask, for the record. A person who answered is named by the - * approver; no answer (timeout, failure, cancelled run) means the policy's - * default decided. A bare true/false names nobody: decidedBy null. + * approver; anything else (no answer in time, an answer naming nobody, a + * broken approver, a cancelled run) means the policy's default decided. * Rule-decided calls carry no decidedBy here; the record writer fills in * "policy" for any decision whose rule list is not `ask`. */ const decider = (approval) => { if (approval.verdict === 'approved' || approval.verdict === 'rejected') { return { - decidedBy: approval.by ?? null, - decidedByRole: approval.role ?? null, + decidedBy: approval.by, + decidedByRole: approval.role, ...(approval.decisionCardId ? { decisionCardId: approval.decisionCardId } : {}), }; } diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 3e5b955065..040c7b2236 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -218,7 +218,10 @@ test('permission.decided: decidedBy is separate from the agent owner, and creden }); assert.deepEqual(asked.actor, { type: 'human', id: 'u-ben' }, 'the decider is the actor, not the agent or its owner'); assert.equal(asked.data.decidedByRole, 'credential-owner'); - for (const role of ['decision-owner', 'agent-owner', 'policy']) decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: role }); + for (const role of ['decision-owner', 'agent-owner']) decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: role }); + decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'policy' }); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'policy' }), /exactly when/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'credential-owner' }), /exactly when/); assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-ben', decidedByRole: 'owner' }), /decidedByRole/); // A parked call is decided by an owner or by policy, never a pod member (ADR-033 D5). assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't', decidedBy: 'u-cal', decidedByRole: 'pod-member' }), /decidedByRole/); @@ -227,9 +230,13 @@ test('permission.decided: decidedBy is separate from the agent owner, and creden assert.equal(byRule.data.decidedBy, 'policy'); assert.equal(byRule.data.decidedByRole, 'policy'); assert.deepEqual(byRule.actor, { type: 'policy', id: 'grant:g-1' }); - const unnamed = decide({ outcome: 'selected', optionId: 'allow', toolCallId: 't', rule: { list: 'ask', pattern: 'x' }, approval: 'approved' }); - assert.equal(unnamed.data.decidedBy, null, 'an ask with no named approver stays null'); - assert.equal(unnamed.actor, null); + // A decision always names who made it; no writer may record one nobody can be held to. + const asked2 = { outcome: 'selected', optionId: 'allow', toolCallId: 't', rule: { list: 'ask', pattern: 'x' }, approval: 'approved' }; + assert.throws(() => decide(asked2), /decidedBy is required/); + assert.throws(() => decide({ ...asked2, decidedBy: null, decidedByRole: null }), /decidedBy is required/); + assert.throws(() => decide({ ...asked2, decidedBy: 'u-ben', decidedByRole: null }), /decidedByRole/); + assert.throws(() => decide({ ...asked2, decidedBy: 'u-ben' }), /decidedByRole/); + assert.throws(() => decide({ outcome: 'cancelled', toolCallId: 't' }), /decidedBy is required/, 'no rule, no decider'); }); test('tool.call surface is one of connector, commonly, native', () => { @@ -307,7 +314,7 @@ test('argsDigest is sha256 of the canonical args, and anything else is refused, assert.equal(argsDigestOf({ s: '\ud800' }), undefined, 'a lone surrogate has no digest (RFC 8785)'); assert.equal(argsDigestOf({ ['k\udc00']: 1 }), undefined, 'in a key too'); const call = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'tool.call', data: { toolCallId: 't', title: 'x', argsDigest } }); - const decided = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data: { outcome: 'cancelled', toolCallId: 't', argsDigest } }); + const decided = (argsDigest) => appendEntry(null, { ...ENV, ts: ts(), kind: 'permission.decided', data: { outcome: 'cancelled', toolCallId: 't', decidedBy: 'policy', decidedByRole: 'policy', argsDigest } }); for (const write of [call, decided, (d) => request(askData({ argsDigest: d }))]) { write(digest); for (const raw of [{ command: 'git push' }, 'git push origin main', `sha256:${'A'.repeat(64)}`, `sha256:${'a'.repeat(63)}`, `sha512:${'a'.repeat(64)}`, 'a'.repeat(64), ['x'], 42]) { diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js index 886e87618e..1d4575689d 100644 --- a/packages/seats/test/record-valet.test.js +++ b/packages/seats/test/record-valet.test.js @@ -105,14 +105,41 @@ const ask = async (answer) => { return { result, data: events[0].data }; }; -test('an approver approves only with true, or an object whose approved is exactly true', async () => { - for (const answer of [true, { approved: true }, { approved: true, by: 'u-ben' }]) { - assert.deepEqual((await ask(answer)).result, { outcome: 'selected', optionId: 'allow' }, JSON.stringify(answer)); +const BEN = { by: 'u-ben', role: 'credential-owner' }; + +test('an approver approves only with approved exactly true and a named owner', async () => { + for (const role of ['credential-owner', 'agent-owner']) { + assert.deepEqual((await ask({ approved: true, by: 'u-ben', role })).result, { outcome: 'selected', optionId: 'allow' }, role); + } + for (const approved of ['yes', 1, 'true', {}, [true], null, undefined]) { + const { result, data } = await ask({ approved, ...BEN }); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }, `approved ${JSON.stringify(approved)} must not approve`); + assert.equal(data.approval, 'failed'); } - for (const answer of [{ approved: 'yes' }, { approved: 1 }, { approved: 'true' }, { approved: {} }, { approved: [true] }, 'yes', 1, {}, [true], null, undefined, false]) { +}); + +test('a bare true approves nothing: the ask fails and policy denies (CTO, 10-25)', async () => { + const { result, data } = await ask(true); + assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }); + assert.deepEqual([data.approval, data.decidedBy, data.decidedByRole], ['failed', 'policy', 'policy']); +}); + +test('a malformed answer approves nothing: the ask fails and policy denies; no decided line names nobody', async () => { + const malformed = [ + 'yes', 1, false, null, undefined, [true], {}, + { approved: true }, + { approved: true, by: 'u-ben' }, + { approved: true, role: 'credential-owner' }, + { approved: true, by: 'Ben Smith', role: 'credential-owner' }, + { approved: true, by: 'u-ben', role: 'boss' }, + { approved: true, by: 'u-ben', role: 'policy' }, + { approved: true, by: 'u-ben', role: null }, + { approved: false }, + ]; + for (const answer of malformed) { const { result, data } = await ask(answer); assert.deepEqual(result, { outcome: 'selected', optionId: 'reject' }, `${JSON.stringify(answer)} must not approve`); - assert.equal(data.approval, 'rejected'); + assert.deepEqual([data.approval, data.decidedBy, data.decidedByRole], ['failed', 'policy', 'policy'], JSON.stringify(answer)); } }); @@ -124,14 +151,12 @@ test('the decision names who decided and as what, separate from the agent and it assert.equal(data.decisionCardId, 'card-1'); assert.equal(data.grantId, 'g-9'); const refused = await ask({ approved: false, by: 'u-ben', role: 'credential-owner' }); + assert.equal(refused.data.approval, 'rejected'); assert.equal(refused.data.decidedBy, 'u-ben', 'a refusal is attributed too'); - // Malformed attribution is dropped rather than trusted. - const odd = await ask({ approved: true, by: 'Ben Smith', role: 'boss', decisionCardId: '' }); - assert.equal(odd.data.decidedBy, null); - assert.equal(odd.data.decidedByRole, null); + // A malformed card id is dropped rather than trusted. + const odd = await ask({ approved: true, ...BEN, decisionCardId: '' }); + assert.equal(odd.data.decidedBy, 'u-ben'); assert.equal('decisionCardId' in odd.data, false); - const bare = await ask(true); - assert.equal(bare.data.decidedBy, null, 'a bare true names nobody'); }); test('a pod member\'s answer decides nothing: the call stays parked until the owner answers (ADR-033 D5)', async () => { @@ -198,7 +223,7 @@ test('the question is redacted: a credential value in the ask never reaches disk const env = await prepareRunEnv({ grant, resolveSecret: async () => FROM_RESOLVER, root, trace: run }); await env.cleanup(); const decide = createPermissionPolicy(grant, { - trace: run, grantId: 'g-3', askedOf: { id: 'u-ben', role: 'credential-owner' }, approve: async () => ({ approved: true, by: 'u-ben' }), + trace: run, grantId: 'g-3', askedOf: { id: 'u-ben', role: 'credential-owner' }, approve: async () => ({ approved: true, ...BEN }), }); await decide({ toolCall: { toolCallId: 'tc-1', title: `Call the API with token ${FROM_RESOLVER}?`, kind: 'fetch' }, options: OPTIONS }); const raw = readFileSync(join(root, 'trace', 'runs', 'run-va-0003.jsonl'), 'utf8'); diff --git a/packages/seats/test/valet-permissions.test.js b/packages/seats/test/valet-permissions.test.js index c528c0f8e5..7f0611d2b4 100644 --- a/packages/seats/test/valet-permissions.test.js +++ b/packages/seats/test/valet-permissions.test.js @@ -107,22 +107,24 @@ test('deny selects reject_once, or cancelled when there is no reject_once option ['selected', 'deny'], ['cancelled', 'deny'], ['selected', 'default'], ['cancelled', 'default']]); }); +const BEN = { by: 'u-ben', role: 'credential-owner' }; + test('ask: approved allows once, rejected denies, and approve sees the tool call and rule', async () => { const seen = []; const trace = recorder(); - let verdict = true; + let verdict = { approved: true, ...BEN }; const decide = createPermissionPolicy({ tools: { ask: ['execute'] } }, { trace, approve: async (input) => { seen.push(input); return verdict; }, }); const req = request('execute', 'npm test'); assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-once' }); - verdict = false; + verdict = { approved: false, ...BEN }; assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }); - verdict = 'yes'; + verdict = { approved: 'yes', ...BEN }; assert.deepEqual(await decide(req), { outcome: 'selected', optionId: 'opt-reject' }, 'only literal true approves'); assert.deepEqual(seen[0], { toolCall: req.toolCall, grantRule: { list: 'ask', pattern: 'execute' } }); - assert.deepEqual(trace.events.map((e) => [e.data.kind, e.data.approval]), [ - ['allow_once', 'approved'], ['reject_once', 'rejected'], ['reject_once', 'rejected']]); + assert.deepEqual(trace.events.map((e) => [e.data.kind, e.data.approval, e.data.decidedBy]), [ + ['allow_once', 'approved', 'u-ben'], ['reject_once', 'rejected', 'u-ben'], ['reject_once', 'failed', 'policy']]); }); test('ask: no answer in time denies; a throwing or missing approver denies', async () => { From fb686d4fdd3143d5fa5f4fbeb43f46c38c3cbf90 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:38:14 -0700 Subject: [PATCH 16/20] test(seats): the pod-member test names who the ask goes to, and its wait is bounded permission.requested is written only when the policy is told who a call is asked of. The test omitted askedOf, so it asserted a requested line the policy was never going to write. Its wait for the second ask was also unbounded, so a policy that let the pod member decide hung the suite instead of failing it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/test/record-valet.test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/seats/test/record-valet.test.js b/packages/seats/test/record-valet.test.js index 1d4575689d..bd649809f5 100644 --- a/packages/seats/test/record-valet.test.js +++ b/packages/seats/test/record-valet.test.js @@ -166,13 +166,16 @@ test('a pod member\'s answer decides nothing: the call stays parked until the ow const ownerAnswer = new Promise((resolve) => { ownerAnswers = resolve; }); const decide = createPermissionPolicy({ tools: { ask: ['fetch'] } }, { trace: { emit: (type, data) => events.push({ type, data }) }, approveTimeoutMs: 5000, grantId: 'g-9', + askedOf: { id: 'u-ben', role: 'credential-owner' }, approve: (input) => { seen.push(input); return seen.length === 1 ? { approved: true, by: 'u-cal', role: 'pod-member', decisionCardId: 'card-1' } : ownerAnswer; }, }); const pending = decide({ toolCall: { toolCallId: 'tc-1', title: 'fetch it', kind: 'fetch' }, options: OPTIONS }); - while (seen.length < 2) await new Promise((resolve) => { setTimeout(resolve, 5); }); + // Bounded: a policy that lets the pod member decide never asks again. + for (let i = 0; i < 200 && seen.length < 2; i += 1) await new Promise((resolve) => { setTimeout(resolve, 5); }); + assert.equal(seen.length, 2, 'the owner is asked again after the pod member\'s answer'); assert.deepEqual(events.map((e) => e.type), ['permission.requested'], 'no permission.decided for the pod member\'s answer'); assert.deepEqual(seen[1].refused, { by: 'u-cal', role: 'pod-member', reason: 'not_owner' }, 'the card is told why'); assert.equal(seen[0].refused, undefined); From 710b91ba1badd02670b434f8a0e56bf784832f0b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:04:48 -0400 Subject: [PATCH 17/20] feat(seats): supervised runs forward actor and redact to the record (#2701) * feat(seats): supervise a run so a hung turn is named stuck and always settles superviseRun opens a traced run, hands the adapter a progress-counting trace and an AbortSignal it owns, and reads liveness from trace events plus the stdout/stderr bytes of any child reported through ctx.onSpawn. No signal for stallMs (120 s) emits liveness.stalled, cancels with a stuck RunFailure, and if the adapter has not settled within cancelGraceMs (10 s) kills the children it saw and settles the run itself. timeoutMs (15 min, per attempt) takes the same path as timeout. Retry is opt-in, honours FAILURE_KINDS.retryable over the caller's list (refusal is never retried), reuses the runId, and carries a failed attempt's sessionId. Exactly one run.settled per run. observeSpawn now passes the child as a second argument to onSpawn so the supervisor can watch and kill it; existing receivers read only the first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(seats): the supervisor's trace wrapper forwards the actor and redact The wrapper a supervised adapter receives as ctx.trace forwarded only emit(type, data). It dropped the third argument, so an adapter's entries lost the actor it named, and it had no redact(), so prepareRunEnv, which registers resolved credential values through traceOf(ctx), handed them to nothing. A value the agent echoed into text, a tool result, a permission question or its final failure was written to the record verbatim. The wrapper now passes emit's options through and offers redact() when the run has one. The store scrubs at write time, before the line is hashed, so the value never reaches disk or the OTLP export, and verifyRun stays ok. The new end-to-end test runs the real supervisor, valet and store against the local fake provider with a fake key, reads every file the store wrote and the OTLP export, and asserts no form of the value appears. Mutations (wrapper drops redact, wrapper drops the actor, the valet passes no values, the export reads unredacted entries) each turn it red. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * test(seats): plant the credential in a permission question asked through the valet Record v1 now requires permission.requested to carry the human-readable question and askedOf. The end-to-end test asks through the real createPermissionPolicy on the supervisor's wrapper, with the credential in the tool call's title and rawInput, and asserts the recorded question, the exported seats.question attribute and every file the store wrote hold [redacted], with the arguments kept as a digest only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(seats): scrub split and short credential values from the record Answers the security gate's HOLD on #2701. - A value split across streamed deltas matched none of them. The store now scrubs agent.message_chunk and agent.thought_chunk text per stream (kind + actor) over a rolling window: it holds back the longest tail that could still begin a known value, joins it to the stream's next delta, and writes any held tail before attempt.started, attempt.settled and run.settled, so held text is never lost and never written whole. - The store scrubs every registered value whatever its length; the eight-character floor stays with the refusal-text reader (MIN_CREDENTIAL_LENGTH). prepareRunEnv refuses a credential under eight characters as credential_denied (credential.denied reason too_short). - The supervisor registers ctx.runtimeToken with the store, and failureData scrubs a RunFailure's detail of the spawn's credentials. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(seats): a stream that ends mid-value writes the marker, never the held prefix A held tail is held only while it is a prefix of a known credential, so at stream end it is part of one. flushStreams now writes `[redacted]` in its place before the next attempt.started / attempt.settled / run.settled. record-v1.md rule 1 says so, including that a crashed writer loses the tail. Tests: end-of-stream and kill-mid-value cases assert the marker and that no prefix bytes reach the record; a value cut one character short is new. Container: seats 170/170; mutant (flush writes the held tail) fails 3. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * chore(secret-scan): allowlist the fake OpenRouter key in the seats supervisor test packages/seats/test/supervisor.test.js:299 uses a value shaped like an OpenRouter key so the scrub test can prove a key never reaches the record. gitleaks' generic-api-key flags it in commit 9fd4e5f, which the scan reads, so an inline allow on today's line would not clear it. The exact literal only; the test is unchanged. gitleaks 8.21.2 with this config: line 299 clear, a different fake key still flagged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- .gitleaks.toml | 5 + packages/seats/README.md | 2 +- packages/seats/docs/record-v1.md | 20 +- packages/seats/src/index.js | 1 + packages/seats/src/runtime-observation.js | 4 +- packages/seats/src/supervisor.js | 287 +++++++++++++ packages/seats/src/trace/events.js | 2 +- packages/seats/src/trace/store.js | 80 +++- packages/seats/src/upstream-refusal.js | 12 +- packages/seats/src/valet/env.js | 17 +- .../seats/test/record-redact-stream.test.js | 258 ++++++++++++ .../test/record-redact-supervised.test.js | 186 +++++++++ .../seats/test/supervisor-process.test.js | 105 +++++ packages/seats/test/supervisor.test.js | 394 ++++++++++++++++++ 14 files changed, 1354 insertions(+), 19 deletions(-) create mode 100644 packages/seats/src/supervisor.js create mode 100644 packages/seats/test/record-redact-stream.test.js create mode 100644 packages/seats/test/record-redact-supervised.test.js create mode 100644 packages/seats/test/supervisor-process.test.js create mode 100644 packages/seats/test/supervisor.test.js diff --git a/.gitleaks.toml b/.gitleaks.toml index 60cabd2423..95360d5fcb 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -22,6 +22,11 @@ regexes = [ # signature. generic-api-key matched it, and main's Secret Scan was red on # every push from 2026-10-16. The exact literal only, so a real token still fails. '''eyJhbGciOiJSUzI1NiJ9\.jwt''', + # A fake OpenRouter key in packages/seats/test/supervisor.test.js (#2701): the + # scrub test needs a value shaped like a real key so it can prove the key never + # reaches the record. generic-api-key matched it in a commit the scan reads, so + # an inline allow on today's line would not clear it. The exact literal only. + '''sk-or-v1-abcdefabcdefabcdef''', # The namespaced cloudflared ExternalSecret has `- secretKey: credentials.json` # (#2439) — a Secret KEY NAME (what the tunnel credentials.json is stored under), not # a value. gitleaks' kubernetes-secret-yaml rule captures `secretKey: credentials` (it diff --git a/packages/seats/README.md b/packages/seats/README.md index 8f86eff12d..990cbab2c0 100644 --- a/packages/seats/README.md +++ b/packages/seats/README.md @@ -1,6 +1,6 @@ # packages/seats -The adapters that run an agent CLI as a supervised seat: one prompt in, one CLI run, one reply out. The package holds the contract, the registry, the failure naming, the spawn observation, and the first adapters (`stub`, `hermes`). +The adapters that run an agent CLI as a supervised seat: one prompt in, one CLI run, one reply out. The package holds the contract, the registry, the failure naming, the spawn observation, the run supervisor (`superviseRun`: a turn with no progress signal for `stallMs` is traced as stuck, cancelled, and settled even if its adapter ignores the cancel), and the first adapters (`stub`, `hermes`). It is private and is not published on its own. `@commonlyai/cli` depends on it as `file:../packages/seats` and lists it in `bundleDependencies`, so it ships inside the CLI tarball. diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 335fd202d8..26ab8f459f 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -333,7 +333,7 @@ built in Go runs its tests with `-race` in CI. | kind | data | default actor | |---|---|---| | `credential.granted` | `{names: [NAME]}` | `system` | -| `credential.denied` | `{name, reason}`, where `reason` is `not_granted` or `unresolved` | `system` | +| `credential.denied` | `{name, reason}`, where `reason` is `not_granted`, `unresolved` or `too_short` (a value under 8 characters) | `system` | These cover the environment credentials a run is started with. Values never appear here: `NAME` is an environment variable name matching @@ -434,11 +434,21 @@ What's new or changed in `EVENT_TYPES`: and a writer refuses one that is not name-shaped. - Agent-authored text, such as message chunks and tool titles, belongs to the agent and could repeat a value the agent holds. A writer that knows - the run's credential values replaces every occurrence of 8 characters or - more with `[redacted]` before the line is hashed. + the run's credential values replaces every occurrence of each, whatever + its length, with `[redacted]` before the line is hashed. A writer does + not grant a credential shorter than 8 characters (`credential.denied`, + `too_short`). + - Message and thought chunks are scrubbed per stream, not per line, so a + value split across chunks is caught. A writer holds back a chunk's tail + while it could begin a known value, and writes it with the stream's next + chunk. If the stream ends first, the held tail is a partial value, so the + writer writes `[redacted]` as a chunk of its own in its place, before the + next `attempt.started`, `attempt.settled` or `run.settled`. A short + ordinary tail that happened to begin a value is redacted too. A held tail + lives only in the writer's memory: a writer that crashes before the next + write loses it, and never writes it. - That replacement covers known values only. A value the writer was not - given, or one the agent transformed (base64, split across chunks), passes - through. + given, or one the agent transformed (base64, say), passes through. 2. **Readers ignore unknown fields and skip unknown kinds.** A reader that meets a kind it doesn't know still checks the chain over that line. 3. **`v` bumps only on a breaking change:** diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index fcfd9a0824..6f0be90d3c 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -14,3 +14,4 @@ export { RECORD_VERSION, RECORD_SCHEMA, KINDS, canonicalLine, appendEntry, verifyRecord, readRecord, hashLine, toolSurfaceOf, argsDigestOf, } from './record/format.js'; export { toOtlpJson } from './record/otel.js'; +export { superviseRun, SUPERVISOR_DEFAULTS, DEFAULT_RETRY_ON } from './supervisor.js'; diff --git a/packages/seats/src/runtime-observation.js b/packages/seats/src/runtime-observation.js index 59eb63ee8c..e9c7bfd48d 100644 --- a/packages/seats/src/runtime-observation.js +++ b/packages/seats/src/runtime-observation.js @@ -19,9 +19,11 @@ export const controlsFromArguments = (args, { modelFlag, effortFlag, valueFlags // Called at the child's successful OS spawn event, not when a requested // configuration is saved or when a turn is merely queued. Never send argv. +// The child rides along as a second argument so a supervisor can read its +// output as a liveness signal and kill it; a reporter reads only the first. export const observeSpawn = (child, onSpawn, controls) => { if (typeof onSpawn !== 'function') return; child.once('spawn', () => { - try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }); } catch { /* diagnostic only */ } + try { onSpawn({ model: controls.model || null, effort: controls.effort || null, at: new Date().toISOString() }, child); } catch { /* diagnostic only */ } }); }; diff --git a/packages/seats/src/supervisor.js b/packages/seats/src/supervisor.js new file mode 100644 index 0000000000..513a420432 --- /dev/null +++ b/packages/seats/src/supervisor.js @@ -0,0 +1,287 @@ +/** + * The run supervisor: one prompt, one traced run, and a guarantee that the run + * settles. A turn that hangs is named `stuck` in the trace, cancelled, and if + * the adapter will not let go, abandoned and settled anyway. Nothing this + * module starts is allowed to sit in "running" forever. + * + * Liveness is read from progress signals, not from the reply: any trace event + * the adapter emits, and any byte on the stdout/stderr of a child the adapter + * reports through `ctx.onSpawn`. A run that is busy but quiet past the window + * is indistinguishable from a hung one, which is why the window is long. + */ +import { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; +import { redactKnownCredentials, scrubAdapterFailure, spawnCredentials } from './upstream-refusal.js'; + +export const SUPERVISOR_DEFAULTS = Object.freeze({ + // Longer than the quiet stretches a healthy turn has: a long-context first + // token, or a tool (an install, a test suite) whose output the CLI buffers. + stallMs: 120_000, + // Covers an adapter's own SIGTERM -> 5 s -> SIGKILL ladder with room to spare. + cancelGraceMs: 10_000, + // The turn budget the run loop already enforces (COMMONLY_AGENT_RUN_TIMEOUT_MS's + // fallback), so supervising a turn does not shorten or lengthen it. + timeoutMs: 15 * 60 * 1000, +}); + +export const DEFAULT_RETRY_ON = Object.freeze(['stuck', 'exited', 'timeout']); + +// setTimeout overflows past this and fires immediately. +const MAX_TIMER_MS = 2_147_483_647; +// The supervisor's own events; an adapter writing them would forge the run's outcome. +const SUPERVISOR_EVENTS = new Set(['run.started', 'run.settled']); + +const windowOf = (value, fallback) => { + if (value === undefined || value === null) return fallback; + const number = Number(value); + if (number === 0 || number === Infinity) return null; // explicitly off + if (!Number.isFinite(number) || number < 0) throw new Error(`supervisor window must be a non-negative number, got ${value}`); + return Math.min(number, MAX_TIMER_MS); +}; + +const resolvePolicy = (policy = {}) => { + const retry = policy.retry || {}; + return { + stallMs: windowOf(policy.stallMs, SUPERVISOR_DEFAULTS.stallMs), + cancelGraceMs: windowOf(policy.cancelGraceMs, SUPERVISOR_DEFAULTS.cancelGraceMs) ?? 0, + timeoutMs: windowOf(policy.timeoutMs, SUPERVISOR_DEFAULTS.timeoutMs), + retryMax: Math.max(0, Math.floor(Number(retry.max) || 0)), + retryOn: Array.isArray(retry.on) ? retry.on : DEFAULT_RETRY_ON, + onStalled: typeof policy.onStalled === 'function' ? policy.onStalled : null, + }; +}; + +const isChild = (value) => !!value && typeof value.kill === 'function' && typeof value.once === 'function'; +const isAlive = (child) => child.exitCode === null && child.signalCode === null; + +// The retryable flag in FAILURE_KINDS outranks the caller's list: a refusal +// listed in `on` is still never re-sent. +const shouldRetry = (failure, attempt, settings) => { + const kind = failureKindOf(failure); + return attempt <= settings.retryMax + && !!kind && FAILURE_KINDS[kind].retryable + && settings.retryOn.includes(kind) + && !failure.adapterIgnoredCancel; +}; + +const outcomeOf = (failure) => { + if (!failure) return 'ok'; + const kind = failureKindOf(failure); + if (kind === 'stuck') return 'stuck'; + if (kind === 'cancelled') return 'cancelled'; + return 'failed'; +}; + +// An adapter that threw a bare Error broke the RunFailure rule; its text is +// still scrubbed before it reaches the trace, and it carries no kind rather +// than a guessed one. A RunFailure's detail keeps its shape but loses any +// credential this spawn was handed, whoever built it. +const failureData = (failure, credentials) => (failure instanceof RunFailure + ? { kind: failure.kind, detail: redactKnownCredentials(failure.detail, credentials) } + : { kind: null, detail: scrubAdapterFailure(String(failure?.message ?? failure ?? ''), { credentials }).detail }); + +const sessionIdOf = (failure) => { + const id = failure?.sessionId ?? failure?.newSessionId; + return typeof id === 'string' && id ? id : null; +}; + +const ignoredCancel = (reason, graceMs) => { + const failure = new RunFailure(reason.kind, `${reason.detail}; the adapter did not settle within ${graceMs}ms of cancellation`); + failure.adapterIgnoredCancel = true; + return failure; +}; + +const callerCancel = (signal) => (failureKindOf(signal.reason) === 'cancelled' + ? signal.reason : runFailure('cancelled', 'cancelled by the caller')); + +/** One attempt. Resolves { result } or { failure }; never rejects, never stays pending. */ +const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, external }) => new Promise((resolve) => { + const controller = new AbortController(); + const children = new Set(); + let live = true; + let abortedBy = null; + let lastSignal = Date.now(); + let stallTimer = null; + let wallTimer = null; + let graceTimer = null; + + const touch = () => { if (live) lastSignal = Date.now(); }; + + const killChildren = () => { + let killed = 0; + for (const child of children) { + if (!isAlive(child)) continue; + try { child.kill('SIGKILL'); killed += 1; } catch { /* already gone */ } + } + return killed; + }; + + const onExternal = () => cancel(callerCancel(external), { recovery: false }); + + const finish = (outcome) => { + if (!live) return; + live = false; + clearTimeout(stallTimer); + clearTimeout(wallTimer); + clearTimeout(graceTimer); + external?.removeEventListener?.('abort', onExternal); + // An adapter that honoured the abort can still leave its child behind. + if (abortedBy && !outcome.failure?.adapterIgnoredCancel && killChildren() > 0) { + run.emit('liveness.recovery', { action: 'kill', attempt }); + } + resolve(outcome); + }; + + function cancel(reason, { recovery = true } = {}) { + if (abortedBy || !live) return; + abortedBy = reason; + clearTimeout(stallTimer); + clearTimeout(wallTimer); + if (recovery) run.emit('liveness.recovery', { action: 'cancel', attempt }); + controller.abort(reason); + graceTimer = setTimeout(() => { + if (!live) return; + run.emit('liveness.recovery', { action: 'kill', attempt }); + killChildren(); + finish({ failure: ignoredCancel(reason, settings.cancelGraceMs) }); + }, settings.cancelGraceMs); + } + + const checkStall = () => { + if (!live || abortedBy) return; + const idleMs = Date.now() - lastSignal; + if (idleMs < settings.stallMs) { + stallTimer = setTimeout(checkStall, settings.stallMs - idleMs); + return; + } + const windowMs = settings.stallMs; + run.emit('liveness.stalled', { idleMs, windowMs }); + if (settings.onStalled) { + // A host's notifier failing must not keep the run from recovering. + try { + Promise.resolve(settings.onStalled({ runId: run.runId, adapter: adapter.name, attempt, idleMs, windowMs })).catch(() => {}); + } catch { /* notifier only */ } + } + cancel(runFailure('stuck', `no progress signal for ${idleMs}ms (window ${windowMs}ms)`)); + }; + + // Counts as progress only while this attempt is live, so a cancelled + // attempt's late output cannot keep the next one alive or reach the trace. + // The emit options (the entry's actor) pass through unchanged. `redact` is + // offered only when the run has one, so prepareRunEnv cannot hand its + // values to a no-op; it is not gated on `live`, since adding a value to + // scrub is never unsafe. + const trace = { + emit(type, data, options) { + if (!live || SUPERVISOR_EVENTS.has(type)) return null; + touch(); + return run.emit(type, data, options); + }, + ...(typeof run.redact === 'function' ? { redact: (values) => run.redact(values) } : {}), + }; + + // Wraps the caller's onSpawn: every argument reaches it unchanged. + const onSpawn = (...args) => { + if (live) { + touch(); + for (const child of args.filter(isChild)) { + children.add(child); + child.stdout?.on?.('data', touch); + child.stderr?.on?.('data', touch); + } + } + return typeof ctx.onSpawn === 'function' ? ctx.onSpawn(...args) : undefined; + }; + + if (external?.aborted) { + abortedBy = callerCancel(external); + finish({ failure: abortedBy }); + return; + } + external?.addEventListener?.('abort', onExternal, { once: true }); + if (settings.stallMs) stallTimer = setTimeout(checkStall, settings.stallMs); + if (settings.timeoutMs) { + wallTimer = setTimeout(() => cancel(runFailure('timeout', `exceeded the ${settings.timeoutMs}ms wall-clock budget`)), settings.timeoutMs); + } + + let pending; + try { + pending = Promise.resolve(adapter.spawn(prompt, { + ...ctx, ...(sessionId ? { sessionId } : {}), trace, signal: controller.signal, onSpawn, + })); + } catch (error) { + pending = Promise.reject(error); + } + // Once the supervisor has aborted, the abort's reason is the outcome, even + // if the adapter answers inside the grace window: the trace already says why. + pending.then( + (result) => finish(abortedBy ? { failure: abortedBy } : { result }), + (error) => finish({ failure: abortedBy || error }), + ); +}); + +/** + * Run one prompt under liveness supervision. + * + * @param {object} args + * @param {import('./contract.js').Adapter} args.adapter + * @param {string} args.prompt + * @param {object} [args.ctx] passed to adapter.spawn; `trace`, `signal` and `onSpawn` are wrapped. + * `ctx.signal`, when given, cancels the run (outcome `cancelled`). + * @param {ReturnType<import('./trace/store.js').createTraceStore>} args.store + * @param {object} [args.policy] + * @param {number} [args.policy.stallMs] no-progress window; 0 turns it off + * @param {number} [args.policy.cancelGraceMs] how long a cancelled adapter has to settle + * @param {number} [args.policy.timeoutMs] wall-clock budget per attempt; 0 turns it off + * @param {{max: number, on?: string[]}} [args.policy.retry] off unless max > 0 + * @param {(info: {runId, adapter, attempt, idleMs, windowMs}) => void} [args.policy.onStalled] + * @param {string} [args.runId] + * @param {string} [args.label] + * @returns {Promise<{ runId: string, outcome: string, attempts: number, result?: object, failure?: Error }>} + */ +export const superviseRun = async ({ adapter, prompt, ctx = {}, store, policy = {}, runId, label } = {}) => { + if (!adapter || typeof adapter.spawn !== 'function') throw new Error('superviseRun needs an adapter with spawn()'); + if (!store || typeof store.startRun !== 'function') throw new Error('superviseRun needs a trace store'); + const settings = resolvePolicy(policy); + const run = store.startRun({ runId, adapter: adapter.name, label }); + // The runtime token rides in ctx, not through the valet, so the store would + // not otherwise know to scrub it from agent text. + if (typeof run.redact === 'function' && typeof ctx.runtimeToken === 'string' && ctx.runtimeToken) { + run.redact([ctx.runtimeToken]); + } + const startedAt = Date.now(); + const external = ctx.signal; + let attempt = 0; + let sessionId = ctx.sessionId || null; + let last = { failure: null }; + + try { + for (;;) { + attempt += 1; + run.emit('run.started', { adapter: adapter.name, ...(sessionId ? { sessionId } : {}), attempt }); + last = await runAttempt({ adapter, prompt, ctx, run, settings, attempt, sessionId, external }); + if (!last.failure || external?.aborted || !shouldRetry(last.failure, attempt, settings)) break; + sessionId = sessionIdOf(last.failure) || sessionId; + run.emit('liveness.recovery', { action: 'retry', attempt: attempt + 1 }); + } + } catch (error) { + // The supervisor's own fault (a trace write failing, say) still settles the run. + last = { failure: error }; + } + + const outcome = outcomeOf(last.failure); + const credentials = spawnCredentials(ctx, [adapter.providerKeyEnv].filter(Boolean)); + try { + run.emit('run.settled', { + outcome, + ...(last.failure ? { failure: failureData(last.failure, credentials) } : {}), + durationMs: Date.now() - startedAt, + attempts: attempt, + }); + } catch { /* the outcome is still returned */ } + return { + runId: run.runId, + outcome, + attempts: attempt, + ...(last.failure ? { failure: last.failure } : { result: last.result }), + }; +}; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index 2069eac6d0..0ababca921 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -23,7 +23,7 @@ export const EVENT_TYPES = Object.freeze([ 'liveness.stalled', // { idleMs, windowMs } 'liveness.recovery', // { action: 'cancel'|'kill'|'retry', attempt } 'credential.granted', // { names: [...] } names only, never values - 'credential.denied', // { name, reason?: 'not_granted'|'unresolved' } + 'credential.denied', // { name, reason?: 'not_granted'|'unresolved'|'too_short' } 'process.exited', // { code, signal } // Record kinds with no earlier trace name (docs/record-v1.md has the full data). 'task.claimed', // { leaseUntil? } mirrors the board's claim; the board stays the source of truth diff --git a/packages/seats/src/trace/store.js b/packages/seats/src/trace/store.js index 4608844385..5049f8d82a 100644 --- a/packages/seats/src/trace/store.js +++ b/packages/seats/src/trace/store.js @@ -14,7 +14,6 @@ import { join } from 'node:path'; import { appendEntry, hashLine, kindOfType, readRecord, traceTypeOf, validateActor, verifyRecord, BOARD_ACTOR, } from '../record/format.js'; -import { redactKnownCredentials } from '../upstream-refusal.js'; const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; const readJsonl = (path) => (existsSync(path) @@ -23,10 +22,22 @@ const readJsonl = (path) => (existsSync(path) }) : []); -/** Known credential values never reach a line, even inside agent text. */ +/** + * Known credential values never reach a line, even inside agent text. Every + * registered value is replaced, whatever its length: the eight-character + * floor in redactKnownCredentials belongs to the refusal-text reader, not to + * the record. `secrets` is kept longest first, so a value that contains a + * shorter one is replaced whole rather than in pieces. + */ +const scrubText = (text, secrets) => { + let out = text; + for (const value of secrets) out = out.split(value).join('[redacted]'); + return out; +}; + const scrub = (value, secrets) => { if (secrets.length === 0) return value; - if (typeof value === 'string') return redactKnownCredentials(value, secrets); + if (typeof value === 'string') return scrubText(value, secrets); if (Array.isArray(value)) return value.map((item) => scrub(item, secrets)); if (value && typeof value === 'object') { return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, scrub(item, secrets)])); @@ -34,6 +45,34 @@ const scrub = (value, secrets) => { return value; }; +/** + * Streamed agent text arrives in deltas, so a value can be split across two or + * more of them and match none on its own. Each stream (kind + actor) is + * scrubbed over a rolling window instead: the held tail is joined to the next + * delta, the join is scrubbed, and the longest suffix of the result that + * could still begin a known value is held back again. What is written is the + * rest. + * + * Why that is enough: an occurrence that started inside written text and ran + * past its end would make the written part plus the held tail a proper prefix + * of that value, a longer such suffix than the one held. So no value spans a + * written boundary, and each written piece is itself scrubbed. The tail is + * shorter than the longest value, and only as long as an actual prefix match. + */ +const STREAMED_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']); +// The held text is written out before these, so it stays in its own attempt +// and is never lost at settle, whichever path (ok, failure, cancel, kill) got there. +const FLUSH_BEFORE = new Set(['attempt.started', 'attempt.settled', 'run.settled']); + +const heldLength = (text, secrets) => { + const longest = secrets.length ? secrets[0].length : 0; + for (let start = Math.max(0, text.length - longest + 1); start < text.length; start += 1) { + const tail = text.slice(start); + if (secrets.some((value) => value.length > tail.length && value.startsWith(tail))) return text.length - start; + } + return 0; +}; + /** What readRun has always returned, plus the envelope: `type` is the trace name. */ const asEvent = (entry) => (entry.kind ? { ...entry, type: traceTypeOf(entry.kind) } : entry); @@ -70,11 +109,12 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { let seq = 0; let settled = false; let broken = false; - const secrets = []; + const secrets = []; // longest first; see scrubText + const streams = new Map(); // stream key → { kind, actor, held } const grants = new Map(); // open grants: grantId → { parentGrantId, binding, expiresAt } const startedAt = now(); - const write = (kind, data, actor) => { + const writeLine = (kind, data, actor) => { const line = appendEntry(last, { runId, taskId, podId, ts: now().toISOString(), kind, data: scrub(data, secrets), actor, }, { runActor }); @@ -97,6 +137,31 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { return asEvent(entry); }; + /** A streamed delta's text, less a tail that could begin a known value. */ + const windowed = (kind, data, actor) => { + if (!STREAMED_KINDS.has(kind) || typeof data?.text !== 'string') return data; + const key = JSON.stringify([kind, actor ?? null]); + const joined = scrubText(`${streams.get(key)?.held ?? ''}${data.text}`, secrets); + const cut = joined.length - heldLength(joined, secrets); + if (cut < joined.length) streams.set(key, { kind, actor, held: joined.slice(cut) }); + else streams.delete(key); + return { ...data, text: joined.slice(0, cut) }; + }; + + const write = (kind, data, actor) => writeLine(kind, windowed(kind, data, actor), actor); + + /** + * The stream has ended, so each held tail can no longer complete a value. + * A tail is held only while it is a prefix of a known value, so it is part + * of a credential: write the marker in its place, never the prefix. + */ + const flushStreams = () => { + for (const [key, { kind, actor }] of streams) { + streams.delete(key); + writeLine(kind, { text: '[redacted]' }, actor); + } + }; + const revoke = (grantIds, reason, actor) => (grantIds.length ? write('grant.revoked', { grantIds, reason }, actor) : null); const emit = (type, data = {}, options = {}) => { @@ -104,6 +169,7 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { if (!kind) throw new Error(`unknown trace event type: ${type}`); if (settled) return null; if (broken) throw new Error(`run ${runId}: the record is broken after a failed write`); + if (FLUSH_BEFORE.has(kind)) flushStreams(); const event = write(kind, data, options?.actor); if (kind === 'task.done') { // The board closed the task: every grant bound to it ends, in one write. @@ -142,8 +208,10 @@ export const createTraceStore = ({ root, now = () => new Date() } = {}) => { return revoke([...grants].filter(([, g]) => g.expiresAt && Date.parse(g.expiresAt) <= cutoff).map(([id]) => id), 'expired'); }; + /** Every non-empty string is scrubbed from here on, whatever its length. */ const redact = (values = []) => { - for (const value of values) if (typeof value === 'string' && !secrets.includes(value)) secrets.push(value); + for (const value of values) if (typeof value === 'string' && value && !secrets.includes(value)) secrets.push(value); + secrets.sort((a, b) => b.length - a.length); }; append(indexPath, { diff --git a/packages/seats/src/upstream-refusal.js b/packages/seats/src/upstream-refusal.js index 598edb8a51..12f7f30cca 100644 --- a/packages/seats/src/upstream-refusal.js +++ b/packages/seats/src/upstream-refusal.js @@ -74,6 +74,9 @@ export const bodyFromError = (value) => { return text.replace(STATUS_PREFIX, '').trim(); }; +/** The shortest value the refusal-text reader treats as a credential. */ +export const MIN_CREDENTIAL_LENGTH = 8; + /** * Replace credential values this spawn was handed with a marker. * @@ -82,11 +85,16 @@ export const bodyFromError = (value) => { * value is in the lines around the secret. Values shorter than eight characters * are ignored: redacting a one-character "credential" would corrupt ordinary * prose, and no issued credential is that short. + * + * The floor belongs to this reader only. The trace store scrubs every value + * registered with it, whatever its length, and the valet refuses to grant a + * credential shorter than this (credential_denied), so no granted value is + * ever below the floor. */ export const redactKnownCredentials = (text, credentials = []) => { let out = String(text ?? ''); for (const value of credentials) { - if (typeof value !== 'string' || value.length < 8) continue; + if (typeof value !== 'string' || value.length < MIN_CREDENTIAL_LENGTH) continue; out = out.split(value).join('[redacted]'); } return out; @@ -350,5 +358,5 @@ export const spawnCredentials = (ctx = {}, envNames = []) => { return [ ctx.runtimeToken, ...envNames.map((name) => env[name]), - ].filter((value) => typeof value === 'string' && value.length >= 8); + ].filter((value) => typeof value === 'string' && value.length >= MIN_CREDENTIAL_LENGTH); }; diff --git a/packages/seats/src/valet/env.js b/packages/seats/src/valet/env.js index f819da8118..fe37dfefbe 100644 --- a/packages/seats/src/valet/env.js +++ b/packages/seats/src/valet/env.js @@ -16,6 +16,7 @@ import { homedir, tmpdir } from 'node:os'; import { delimiter, join, resolve, sep } from 'node:path'; import { traceOf } from '../trace/events.js'; import { runFailure } from '../failures.js'; +import { MIN_CREDENTIAL_LENGTH } from '../upstream-refusal.js'; import { validateGrant } from './grant.js'; // Same spirit as the goose adapter's SAFE_ENV: locale, terminal and TLS trust @@ -103,8 +104,10 @@ const grantIssued = (meta) => { * * `require` lists credential names the adapter cannot run without; one the * grant does not name fails the run as credential_denied before any spawn. - * A named credential that cannot be resolved fails it the same way: a run - * that silently lacks its key fails later and less legibly. + * A named credential that cannot be resolved, or that resolves to fewer than + * MIN_CREDENTIAL_LENGTH characters, fails it the same way: a run that + * silently lacks its key fails later and less legibly, and a value that short + * would slip past the refusal-text reader's redaction. * * Events go to `trace`, else ctx.trace: grant.issued (only with grantMeta, * first), then credential.granted {names} or credential.denied {name, reason}. @@ -120,9 +123,14 @@ export const prepareRunEnv = async ({ const issued = grantMeta ? grantIssued(grantMeta) : null; if (issued) hook.emit('grant.issued', issued); const realHomes = [...new Set([baseEnv.HOME, homedir()].filter(Boolean))]; + const DENIED = { + not_granted: 'is not in the run grant', + unresolved: 'could not be resolved', + too_short: `resolved to a value shorter than ${MIN_CREDENTIAL_LENGTH} characters`, + }; const deny = (name, reason, values) => { hook.emit('credential.denied', { name, reason }); - return runFailure('credential_denied', `${name} ${reason === 'not_granted' ? 'is not in the run grant' : 'could not be resolved'}`, { credentials: values }); + return runFailure('credential_denied', `${name} ${DENIED[reason]}`, { credentials: values }); }; const granted = new Set(valid.credentials.map((c) => c.name)); @@ -139,6 +147,9 @@ export const prepareRunEnv = async ({ // A resolver's error text may quote the secret it failed on; drop it. try { value = await resolveOne(spec, { baseEnv, resolveSecret }); } catch { value = undefined; } if (!usable(value)) throw deny(spec.name, 'unresolved', values); + // A value this short cannot be told apart from ordinary text, so the + // refusal-text reader would not redact it. Refuse it here, before spawn. + if (value.length < MIN_CREDENTIAL_LENGTH) throw deny(spec.name, 'too_short', values); values.push(value); env[spec.name] = value; } diff --git a/packages/seats/test/record-redact-stream.test.js b/packages/seats/test/record-redact-stream.test.js new file mode 100644 index 0000000000..ba178a30c3 --- /dev/null +++ b/packages/seats/test/record-redact-stream.test.js @@ -0,0 +1,258 @@ +/** + * The gaps the security gate on #2701 found in the store's redaction, each + * closed with a test that fails on the code it replaced: + * + * 1. a value split across streamed deltas matched none of them; + * 2. a value under eight characters was granted and then not scrubbed; + * 3. the runtime token and a RunFailure's detail never met the scrub. + * + * "Stored text" below is every chunk text the record holds for one stream, + * joined in order: what a reader reconstructs the agent's output from. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createTraceStore } from '../src/trace/store.js'; +import { superviseRun } from '../src/supervisor.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { RunFailure } from '../src/failures.js'; + +const CANARY = 'sk-test-CANARY-7f3a91c2d4e5b6a8'; +const AGENT = { type: 'agent', id: 'u-agent-stream', owner: 'u-owner-stream' }; +const OTHER = { type: 'agent', id: 'u-agent-other', owner: 'u-owner-stream' }; +const FAST = { stallMs: 0, timeoutMs: 0, cancelGraceMs: 40 }; + +const withStore = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-stream-')); + try { return await fn(createTraceStore({ root }), root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const allBytes = (root) => readdirSync(root, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()) + .map((d) => readFileSync(join(d.parentPath ?? d.path, d.name), 'utf8')) + .join('\n'); +const storedText = (store, runId, kind, actorId) => store.readRun(runId) + .filter((e) => e.kind === kind && (actorId === undefined || e.actor?.id === actorId)) + .map((e) => e.data.text) + .join(''); + +/** The checks every split case shares: not in the bytes, not in the joined text, and the marker present. */ +const assertScrubbed = (store, root, runId, streams) => { + assert.equal(allBytes(root).includes(CANARY), false, 'the value is in the record bytes'); + for (const { kind, actorId, expected } of streams) { + const text = storedText(store, runId, kind, actorId); + assert.equal(text.includes(CANARY), false, `the joined ${kind} text reconstructs the value: ${text}`); + assert.equal(text, expected, `the ${kind} stream lost or changed text`); + assert.ok(text.includes('[redacted]'), 'positive control: the scrub replaced the value'); + } + assert.equal(store.verifyRun(runId).ok, true, 'held text is written through the chain like any other line'); +}; + +const splits = (n) => Array.from({ length: n - 1 }, (_, i) => i + 1); + +test('rhea\'s case: a value split across two message deltas is scrubbed', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-split-rhea' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: 'the key is sk-test-CANA' }); + run.emit('agent.message_chunk', { text: 'RY-7f3a91c2d4e5b6a8 end' }); + const texts = store.readRun('run-split-rhea').map((e) => e.data.text); + assert.deepEqual(texts, ['the key is ', '[redacted] end']); + assert.equal(allBytes(root).includes(CANARY), false); +})); + +for (const kind of ['agent.message_chunk', 'agent.thought_chunk']) { + test(`${kind}: a value split in two, at every split point, never reaches the record`, () => withStore(async (store, root) => { + for (const at of splits(CANARY.length)) { + const runId = `run-two-${kind.length}-${at}`; + const run = store.startRun({ runId, actor: AGENT }); + run.redact([CANARY]); + run.emit('run.started', { adapter: 'fake', attempt: 1 }); + run.emit(kind, { text: `the key is ${CANARY.slice(0, at)}` }); + run.emit(kind, { text: `${CANARY.slice(at)} end` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [{ kind, expected: 'the key is [redacted] end' }]); + } + })); +} + +test('a value split three ways, at every pair of split points, never reaches the record', () => withStore(async (store, root) => { + for (const a of splits(CANARY.length)) { + for (let b = a + 1; b < CANARY.length; b += 1) { + const runId = `run-three-${a}-${b}`; + const run = store.startRun({ runId }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `key ${CANARY.slice(0, a)}` }); + run.emit('agent.message_chunk', { text: CANARY.slice(a, b) }); + run.emit('agent.message_chunk', { text: `${CANARY.slice(b)}!` }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [{ kind: 'agent.message_chunk', expected: 'key [redacted]!' }]); + } + } +})); + +test('two streams interleaved, each splitting the value, are scrubbed apart', () => withStore(async (store, root) => { + for (const at of splits(CANARY.length)) { + // Message vs thought, and the same kind from two actors: four streams, one line each in turn. + const runId = `run-mix-${at}`; + const run = store.startRun({ runId, actor: AGENT }); + run.redact([CANARY]); + const head = `a ${CANARY.slice(0, at)}`; + const tail = `${CANARY.slice(at)} z`; + run.emit('agent.message_chunk', { text: head }); + run.emit('agent.thought_chunk', { text: head }); + run.emit('agent.message_chunk', { text: head }, { actor: OTHER }); + run.emit('usage', { inputTokens: 1 }); // a non-stream line between the halves releases nothing + run.emit('agent.thought_chunk', { text: tail }); + run.emit('agent.message_chunk', { text: tail }, { actor: OTHER }); + run.emit('agent.message_chunk', { text: tail }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + assertScrubbed(store, root, runId, [ + { kind: 'agent.message_chunk', actorId: AGENT.id, expected: 'a [redacted] z' }, + { kind: 'agent.message_chunk', actorId: OTHER.id, expected: 'a [redacted] z' }, + { kind: 'agent.thought_chunk', actorId: AGENT.id, expected: 'a [redacted] z' }, + ]); + } +})); + +test('a stream that ends mid-value writes the marker before settle, never the prefix', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-held-settle' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `cut off at ${CANARY.slice(0, 12)}` }); + run.emit('run.settled', { outcome: 'failed', failure: { kind: 'exited', detail: 'gone' }, durationMs: 1, attempts: 1 }); + const events = store.readRun('run-held-settle'); + assert.deepEqual(events.map((e) => [e.kind, e.data.text]), [ + ['agent.message_chunk', 'cut off at '], + ['agent.message_chunk', '[redacted]'], + ['run.settled', undefined], + ]); + assert.equal(allBytes(root).includes(CANARY.slice(0, 12)), false); +})); + +// The security gate's probe: all but the last character, then the run ends. +test('a value cut one character short never reaches the record', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-held-short' }); + run.redact([CANARY]); + run.emit('agent.message_chunk', { text: `key: ${CANARY.slice(0, -1)}` }); + run.emit('liveness.recovery', { action: 'kill', attempt: 1 }); + run.emit('run.settled', { outcome: 'failed', failure: { kind: 'exited', detail: 'killed' }, durationMs: 1, attempts: 1 }); + assert.deepEqual(storedText(store, 'run-held-short', 'agent.message_chunk', undefined), 'key: [redacted]'); + assert.equal(allBytes(root).includes(CANARY.slice(0, -1)), false); + assert.equal(allBytes(root).includes(CANARY.slice(0, 8)), false); +})); + +// The value is registered as the run's runtime token, the one the supervisor itself hands the store. +test('a supervised run killed mid-value writes the marker for its held tail, before the settle and the retry', () => withStore(async (store, root) => { + let spawns = 0; + const adapter = { + name: 'split-then-hang', + spawn(prompt, ctx) { + spawns += 1; + ctx.trace.emit('agent.message_chunk', { text: `try ${spawns}: ${CANARY.slice(0, 10)}` }); + // Ignores the abort, so the supervisor's grace timer kills it. + return new Promise(() => {}); + }, + }; + const ctrl = new AbortController(); + setTimeout(() => ctrl.abort(), 30); + const out = await superviseRun({ + adapter, prompt: 'p', store, ctx: { signal: ctrl.signal, runtimeToken: CANARY }, policy: FAST, + }); + assert.equal(out.outcome, 'cancelled'); + const events = store.readRun(out.runId); + const settledAt = events.findIndex((e) => e.kind === 'run.settled'); + const chunks = events.filter((e) => e.kind === 'agent.message_chunk'); + assert.deepEqual(chunks.map((e) => e.data.text), ['try 1: ', '[redacted]']); + assert.ok(chunks.every((e) => e.seq < events[settledAt].seq), 'the marker landed before run.settled'); + assert.equal(allBytes(root).includes(CANARY.slice(0, 10)), false, 'no prefix of the value on disk'); + + // On retry, the first attempt's tail lands before the second attempt starts, + // not glued to the second's text. (After the retry line: an adapter may still + // stream during a cancel's grace window, so recovery lines do not release.) + let tries = 0; + const retrying = { + name: 'split-then-exit', + async spawn(prompt, ctx) { + tries += 1; + ctx.trace.emit('agent.message_chunk', { text: `try ${tries}: ${CANARY.slice(0, 10)}` }); + if (tries === 1) throw new RunFailure('exited', 'the CLI died'); + return {}; + }, + }; + const again = await superviseRun({ + adapter: retrying, prompt: 'p', store, ctx: { runtimeToken: CANARY }, policy: { ...FAST, retry: { max: 1 } }, + }); + assert.equal(again.outcome, 'ok'); + assert.deepEqual(store.readRun(again.runId).map((e) => e.kind === 'agent.message_chunk' ? e.data.text : e.kind), [ + 'attempt.started', 'try 1: ', 'liveness.recovery', '[redacted]', + 'attempt.started', 'try 2: ', '[redacted]', 'run.settled', + ]); + assert.equal(allBytes(root).includes(CANARY.slice(0, 10)), false); +})); + +test('the store scrubs a registered value of any length', () => withStore(async (store, root) => { + const run = store.startRun({ runId: 'run-short-1' }); + run.redact(['abc123']); + run.emit('agent.message_chunk', { text: 'value abc123 here' }); + run.emit('tool.update', { toolCallId: 't', title: 'abc123' }); + run.emit('run.settled', { outcome: 'ok', durationMs: 1, attempts: 1 }); + const bytes = allBytes(root); + assert.equal(bytes.includes('abc123'), false, 'a six-character value reached the record'); + assert.equal(storedText(store, 'run-short-1', 'agent.message_chunk'), 'value [redacted] here'); +})); + +test('the valet refuses a credential under eight characters as credential_denied, before any HOME', async () => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-short-')); + try { + const events = []; + const trace = { emit: (type, data) => events.push({ type, data }), redact: () => assert.fail('a refused value was registered') }; + await assert.rejects(prepareRunEnv({ + grant: { credentials: [{ name: 'SHORT_KEY', from: 'resolver' }] }, resolveSecret: async () => 'abc123', root, trace, + }), (error) => { + assert.ok(error instanceof RunFailure); + assert.equal(error.kind, 'credential_denied'); + assert.match(error.message, /SHORT_KEY resolved to a value shorter than 8 characters/); + assert.equal(error.message.includes('abc123'), false); + return true; + }); + assert.deepEqual(events, [{ type: 'credential.denied', data: { name: 'SHORT_KEY', reason: 'too_short' } }]); + assert.deepEqual(readdirSync(root), [], 'the refused prepare removed its HOME'); + // Positive control: eight characters is enough. + const ok = await prepareRunEnv({ + grant: { credentials: [{ name: 'SHORT_KEY', from: 'resolver' }] }, resolveSecret: async () => 'abcd1234', root, trace: { emit() {} }, + }); + assert.equal(ok.env.SHORT_KEY, 'abcd1234'); + await ok.cleanup(); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +const TOKEN = 'cm_agent_runtime-token-5e6f7a8b9c0d'; + +test('the run\'s runtime token is scrubbed from agent text', () => withStore(async (store, root) => { + const adapter = { + name: 'token-echo', + async spawn(prompt, ctx) { + ctx.trace.emit('agent.message_chunk', { text: `my token is ${ctx.runtimeToken}.` }); + return {}; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { runtimeToken: TOKEN }, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.equal(allBytes(root).includes(TOKEN), false, 'the runtime token reached the record'); + assert.equal(storedText(store, out.runId, 'agent.message_chunk'), 'my token is [redacted].'); +})); + +test('a RunFailure\'s detail is scrubbed of the spawn\'s credentials before settle', () => withStore(async (store) => { + // A run with no redact hook: only failureData stands between the detail and the line. + const bare = { + startRun: (meta) => { + const { redact, ...run } = store.startRun(meta); + return run; + }, + }; + const adapter = { name: 'leaky-failure', async spawn() { throw new RunFailure('exited', `auth failed for ${TOKEN}`); } }; + const out = await superviseRun({ adapter, prompt: 'p', store: bare, ctx: { runtimeToken: TOKEN }, policy: FAST }); + assert.equal(out.outcome, 'failed'); + const [settled] = store.readRun(out.runId).filter((e) => e.kind === 'run.settled'); + assert.deepEqual(settled.data.failure, { kind: 'exited', detail: 'auth failed for [redacted]' }); +})); diff --git a/packages/seats/test/record-redact-supervised.test.js b/packages/seats/test/record-redact-supervised.test.js new file mode 100644 index 0000000000..5ec6d46f0b --- /dev/null +++ b/packages/seats/test/record-redact-supervised.test.js @@ -0,0 +1,186 @@ +/** + * A credential value the valet resolves for a supervised run never reaches + * the record, even when the agent repeats it. The value travels the real + * path: prepareRunEnv is called with the spawn ctx the supervisor hands the + * adapter, so it registers the value through the supervisor's trace wrapper, + * and the store scrubs it before a line is hashed or written. + * + * Everything is local and keyless: the fake provider answers on 127.0.0.1 and + * the "credential" is a fake value. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { prepareRunEnv } from '../src/valet/env.js'; +import { createPermissionPolicy } from '../src/valet/permissions.js'; +import { toOtlpJson } from '../src/record/otel.js'; +import { runFailure } from '../src/failures.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +const CANARY = 'sk-test-CANARY-7f3a91c2d4e5b6a8'; +const KEY_NAME = 'FAKE_PROVIDER_KEY'; +const GRANT = { credentials: [{ name: KEY_NAME, from: 'resolver' }], tools: { allow: ['read'], ask: ['fetch'] } }; +const AGENT = { type: 'agent', id: 'u-agent-redact', owner: 'u-owner-redact', name: 'Redact test agent' }; +const OWNER = { id: 'u-owner-redact', role: 'agent-owner' }; +// The kinds the adapter writes itself, naming its actor. permission.* are the valet's. +const AGENT_KINDS = new Set(['agent.message_chunk', 'tool.call', 'tool.update', 'result.reported']); + +// The store's redaction is exact-match on known values (docs/record-v1.md §3, +// rule 1): a value the agent transforms, such as base64, is not claimed and +// is not asserted here. The canary is all unreserved characters, so its +// URL-encoded and JSON-escaped forms are the value itself; the check names +// them anyway so a change of canary cannot quietly drop them. +const FORMS = Object.freeze({ + raw: CANARY, + urlEncoded: encodeURIComponent(CANARY), + jsonEscaped: JSON.stringify(CANARY).slice(1, -1), +}); +const assertAbsent = (text, where) => { + for (const [form, value] of Object.entries(FORMS)) { + assert.equal(text.includes(value), false, `the ${form} credential value appears in ${where}`); + } +}; + +const allFiles = (dir) => readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((d) => d.isFile()) + .map((d) => join(d.parentPath ?? d.path, d.name)); + +/** The text an OpenAI-style SSE stream carried, one string per content delta. */ +const sseDeltas = async (response) => { + const body = await response.text(); + return body.split('\n') + .filter((line) => line.startsWith('data: ') && line !== 'data: [DONE]') + .map((line) => JSON.parse(line.slice(6)).choices?.[0]?.delta?.content) + .filter((text) => typeof text === 'string' && text.length > 0); +}; + +/** + * An adapter that does what a CLI adapter does with a granted key: asks the + * valet for its env, calls the provider with it, and streams what comes back. + * The provider is scripted to echo the key, so the key lands in agent text, a + * tool result, the question the valet's permission policy records, and the + * final result. + */ +const echoingAdapter = ({ baseURL, envRoot, fail, seen }) => ({ + name: 'echo-fake', + providerKeyEnv: KEY_NAME, + async spawn(prompt, ctx) { + const prepared = await prepareRunEnv({ + grant: GRANT, resolveSecret: async () => CANARY, root: envRoot, ctx, require: [KEY_NAME], + }); + try { + const key = prepared.env[KEY_NAME]; + seen.key = key; + const as = { actor: AGENT }; + ctx.trace.emit('turn.prompt_sent', { chars: prompt.length }); + const response = await fetch(`${baseURL}/chat/completions`, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, + body: JSON.stringify({ model: 'fake-model', stream: true, messages: [{ role: 'user', content: `[[reply:the key is ${key}]]` }] }), + }); + const deltas = await sseDeltas(response); + seen.reply = deltas.join(''); + for (const text of deltas) ctx.trace.emit('agent.message_chunk', { text }, as); + ctx.trace.emit('tool.call', { + toolCallId: 'tc-1', title: `printenv ${KEY_NAME}`, kind: 'execute', status: 'pending', surface: 'native', + }, as); + ctx.trace.emit('tool.update', { toolCallId: 'tc-1', status: 'completed', title: `${KEY_NAME}=${key}` }, as); + const decide = createPermissionPolicy(GRANT, { + trace: ctx.trace, askedOf: OWNER, approve: async () => ({ approved: true, by: OWNER.id, role: OWNER.role }), + }); + const answer = await decide({ + toolCall: { toolCallId: 'tc-2', title: `Send ${key} to the deploy hook?`, kind: 'fetch', rawInput: { token: key } }, + options: [{ optionId: 'allow', kind: 'allow_once' }, { optionId: 'reject', kind: 'reject_once' }], + }); + seen.answer = answer; + if (fail) throw runFailure('exited', `the CLI exited after printing ${key}`); + ctx.trace.emit('agent.message_chunk', { text: `Done. I used ${key}.` }, as); + ctx.trace.emit('result.reported', { status: 'done', refs: [{ type: 'note', id: `used-${key}` }] }, as); + return { text: seen.reply }; + } finally { + await prepared.cleanup(); + } + }, +}); + +const withHarness = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-redact-sup-')); + const fake = await startFakeProvider(); + try { + const traceRoot = join(root, 'trace'); + return await fn({ fake, root, traceRoot, store: createTraceStore({ root: traceRoot }) }); + } finally { + await fake.close(); + rmSync(root, { recursive: true, force: true }); + } +}; + +for (const fail of [false, true]) { + test(`a granted credential the agent echoes never reaches the record or its export (${fail ? 'failed' : 'ok'} run)`, () => withHarness(async ({ + fake, root, traceRoot, store, + }) => { + const seen = {}; + const out = await superviseRun({ + adapter: echoingAdapter({ baseURL: fake.baseURL, envRoot: root, fail, seen }), + prompt: 'deploy it', + store, + policy: { stallMs: 0, timeoutMs: 0 }, + }); + + // Positive controls: the value really reached the run and the agent really repeated it. + assert.equal(seen.key, CANARY, 'the valet resolved the planted value into the run env'); + assert.ok(seen.reply.includes(CANARY), 'the provider echoed the value back as agent text'); + assert.deepEqual(fake.requests.map((r) => r.authorization), ['present']); + assert.equal(out.outcome, fail ? 'failed' : 'ok'); + if (fail) assert.ok(out.failure.detail.includes(CANARY), 'the failure the run settled on quoted the value'); + + // Every byte the store wrote: the run file and the index. + const files = allFiles(traceRoot); + assert.deepEqual(files.map((f) => f.slice(traceRoot.length + 1)).sort(), ['index.jsonl', `runs/${out.runId}.jsonl`]); + const bytes = files.map((f) => readFileSync(f, 'utf8')).join('\n'); + assertAbsent(bytes, 'the record files'); + + const entries = store.readRun(out.runId); + // The export reads what the store hands back, so it is checked on its own, + // before anything else about those entries. + const otlp = JSON.stringify(toOtlpJson(entries)); + assertAbsent(otlp, 'the OTLP export'); + assert.ok(otlp.includes('[redacted]'), 'positive control: a scrubbed field reached the export'); + assert.ok(otlp.includes('seats.question'), 'positive control: the permission question reached the export'); + assert.ok(otlp.includes('Send [redacted] to the deploy hook?'), 'the exported question is the scrubbed one'); + + const byKind = (kind) => entries.filter((e) => e.kind === kind); + // Each place the value was planted holds the marker instead, so its absence above is a scrub, not a drop. + assert.ok(byKind('agent.message_chunk').some((e) => e.data.text.includes('[redacted]')), 'agent text was scrubbed'); + assert.equal(byKind('tool.update')[0].data.title, `${KEY_NAME}=[redacted]`, 'the tool result was scrubbed'); + const [asked] = byKind('permission.requested'); + assert.equal(asked.data.question, 'Send [redacted] to the deploy hook?', 'the permission question was scrubbed'); + assert.deepEqual(asked.data.askedOf, OWNER); + assert.match(asked.data.argsDigest, /^sha256:[0-9a-f]{64}$/, 'the arguments are recorded as a digest only'); + assert.deepEqual(seen.answer, { outcome: 'selected', optionId: 'allow' }); + const [settled] = byKind('run.settled'); + if (fail) { + assert.equal(settled.data.failure.detail, 'the CLI exited after printing [redacted]', 'the final failure was scrubbed'); + const indexRow = readFileSync(join(traceRoot, 'index.jsonl'), 'utf8').split('\n').filter(Boolean).map(JSON.parse).find((r) => r.outcome); + assert.equal(indexRow.failure.detail, 'the CLI exited after printing [redacted]', 'the index row was scrubbed'); + } else { + assert.equal(byKind('result.reported')[0].data.refs[0].id, 'used-[redacted]', 'the final result was scrubbed'); + assert.ok(byKind('agent.message_chunk').some((e) => e.data.text === 'Done. I used [redacted].')); + } + assert.deepEqual(byKind('credential.granted')[0].data, { names: [KEY_NAME] }, 'the valet recorded the name through the wrapper'); + + // Scrubbed before hashing: the chain over the scrubbed lines is intact. + const verified = store.verifyRun(out.runId); + assert.equal(verified.ok, true, JSON.stringify(verified.errors)); + assert.equal(verified.entries, entries.length); + + // The adapter's own entries carry the actor it named. + const agentEntries = entries.filter((e) => AGENT_KINDS.has(e.kind)); + assert.ok(agentEntries.length >= (fail ? 3 : 5), `${agentEntries.length} adapter entries`); + for (const entry of agentEntries) assert.deepEqual(entry.actor, AGENT, `actor on ${entry.kind} #${entry.seq}`); + })); +} diff --git a/packages/seats/test/supervisor-process.test.js b/packages/seats/test/supervisor-process.test.js new file mode 100644 index 0000000000..cb71038d67 --- /dev/null +++ b/packages/seats/test/supervisor-process.test.js @@ -0,0 +1,105 @@ +// Real processes against the fake provider: the hang is a genuine HTTP call +// that never answers, in a genuine child, so liveness has to fire on silence +// it did not stage, and the child has to actually die. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +// Says it is starting (a byte of progress), then waits on a completion the +// fake provider holds open forever. +const CHILD = ` +process.stdout.write('requesting\\n'); +fetch(process.argv[1] + '/chat/completions', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'fake-model', messages: [{ role: 'user', content: '[[stall]]' }] }), +}).then((r) => r.text()).then((t) => process.stdout.write(t)); +`; + +const nodeFetchAdapter = ({ honourAbort, children }) => ({ + name: 'node-fetch', + spawn(prompt, ctx) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['-e', CHILD, ctx.baseURL], { cwd: ctx.cwd, env: ctx.env, stdio: ['ignore', 'pipe', 'pipe'] }); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + let out = ''; + child.stdout.on('data', (chunk) => { out += chunk; }); + if (honourAbort) ctx.signal.addEventListener('abort', () => child.kill('SIGTERM'), { once: true }); + child.on('close', (code, signal) => { + ctx.trace.emit('process.exited', { code, signal }); + if (ctx.signal.aborted) reject(ctx.signal.reason); + else if (code === 0) resolve({ text: out }); + else reject(runFailure('exited', `node exited ${code ?? signal}`)); + }); + }); + }, +}); + +const isRunning = (pid) => { + try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } +}; + +const withProvider = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-sup-proc-')); + const fake = await startFakeProvider(); + try { + return await fn({ fake, home, store: createTraceStore({ root: join(home, 'trace') }) }); + } finally { + await fake.close(); + rmSync(home, { recursive: true, force: true }); + } +}; + +const settleChild = async (child) => { + if (child.exitCode === null && child.signalCode === null) { + let timer; + await Promise.race([once(child, 'exit'), new Promise((r) => { timer = setTimeout(r, 3000); })]); + clearTimeout(timer); + } +}; + +for (const honourAbort of [false, true]) { + test(`a real child hung on a stalled HTTP call is caught and killed (adapter ${honourAbort ? 'honours' : 'ignores'} abort)`, () => withProvider(async ({ fake, home, store }) => { + const children = []; + const ctx = { baseURL: fake.baseURL, cwd: home, env: { PATH: process.env.PATH, HOME: home } }; + const started = Date.now(); + const out = await superviseRun({ + adapter: nodeFetchAdapter({ honourAbort, children }), prompt: 'p', ctx, store, + policy: { stallMs: 400, cancelGraceMs: 400, timeoutMs: 10000 }, + }); + const elapsed = Date.now() - started; + + assert.equal(out.outcome, 'stuck'); + assert.equal(out.failure.kind, 'stuck'); + assert.ok(elapsed < 3000, `settled in ${elapsed}ms`); + assert.deepEqual(fake.requests.map((r) => r.directive), ['stall'], 'the child really made the call that hung'); + + assert.equal(children.length, 1); + const [child] = children; + await settleChild(child); + assert.ok(child.signalCode, `the child was killed by a signal (exit ${child.exitCode}, signal ${child.signalCode})`); + assert.equal(isRunning(child.pid), false, `pid ${child.pid} is gone`); + + const types = store.readRun(out.runId).map((e) => (e.type === 'liveness.recovery' ? `recovery:${e.data.action}` : e.type)); + if (honourAbort) { + assert.equal(child.signalCode, 'SIGTERM'); + assert.equal(out.failure.adapterIgnoredCancel, undefined); + assert.deepEqual(types, ['run.started', 'liveness.stalled', 'recovery:cancel', 'process.exited', 'run.settled']); + } else { + // Nobody but the supervisor stopped it: the kill came from the handle onSpawn gave it. + assert.equal(child.signalCode, 'SIGKILL'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.deepEqual(types, ['run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:kill', 'run.settled']); + } + })); +} diff --git a/packages/seats/test/supervisor.test.js b/packages/seats/test/supervisor.test.js new file mode 100644 index 0000000000..0c0e806d9b --- /dev/null +++ b/packages/seats/test/supervisor.test.js @@ -0,0 +1,394 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { superviseRun, SUPERVISOR_DEFAULTS, DEFAULT_RETRY_ON } from '../src/supervisor.js'; +import * as seats from '../src/index.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { RunFailure, runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; + +// Small windows instead of a fake clock: every timer here is real, so the +// supervisor is exercised exactly as it runs, just in milliseconds. +const FAST = { stallMs: 60, cancelGraceMs: 60, timeoutMs: 0 }; +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const withStore = async (fn) => { + const root = mkdtempSync(join(tmpdir(), 'seats-sup-')); + try { return await fn(createTraceStore({ root })); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const typesOf = (store, runId) => store.readRun(runId).map((e) => ( + e.type === 'liveness.recovery' ? `recovery:${e.data.action}` : e.type)); +const settledOf = (store, runId) => store.readRun(runId).filter((e) => e.type === 'run.settled'); + +/** Hangs until aborted, then rejects with the signal's reason, as the seam asks. */ +const hangsHonouringAbort = (calls = []) => ({ + name: 'hang-honour', + spawn(prompt, ctx) { + calls.push(ctx); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason), { once: true }); + }); + }, +}); + +/** A fake child: an EventEmitter with stdout/stderr and a kill that records. */ +const fakeChild = () => { + const child = new EventEmitter(); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.exitCode = null; + child.signalCode = null; + child.kills = []; + child.kill = (signal) => { + child.kills.push(signal); + child.signalCode = signal; + setImmediate(() => child.emit('exit', null, signal)); + return true; + }; + return child; +}; + +test('defaults are the documented windows and retry is off', () => { + assert.deepEqual({ ...SUPERVISOR_DEFAULTS }, { stallMs: 120000, cancelGraceMs: 10000, timeoutMs: 900000 }); + assert.deepEqual([...DEFAULT_RETRY_ON], ['stuck', 'exited', 'timeout']); + assert.equal(seats.superviseRun, superviseRun); +}); + +test('a successful run: started, the adapter\'s events, settled ok — in that order', () => withStore(async (store) => { + const adapter = { + name: 'ok', + async spawn(prompt, ctx) { + ctx.trace.emit('turn.prompt_sent', { chars: prompt.length }); + ctx.trace.emit('agent.message_chunk', { text: 'PONG' }); + return { text: 'PONG', newSessionId: 's-1' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'ping', store, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.deepEqual(out.result, { text: 'PONG', newSessionId: 's-1' }); + assert.equal(out.failure, undefined); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'turn.prompt_sent', 'agent.message_chunk', 'run.settled']); + const [settled] = settledOf(store, out.runId); + assert.equal(settled.data.outcome, 'ok'); + assert.equal(settled.data.attempts, 1); + assert.equal(typeof settled.data.durationMs, 'number'); + assert.equal(store.listRuns()[0].outcome, 'ok'); +})); + +test('a hung adapter that honours abort settles stuck, with the reason on its signal', () => withStore(async (store) => { + const calls = []; + const stalled = []; + const out = await superviseRun({ + adapter: hangsHonouringAbort(calls), prompt: 'p', store, + policy: { ...FAST, cancelGraceMs: 2000, onStalled: (info) => stalled.push(info) }, + }); + assert.equal(out.outcome, 'stuck'); + assert.ok(out.failure instanceof RunFailure); + assert.equal(out.failure.kind, 'stuck'); + assert.equal(out.failure.adapterIgnoredCancel, undefined); + assert.equal(calls[0].signal.reason.kind, 'stuck', 'the adapter sees a RunFailure as the abort reason'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'liveness.stalled', 'recovery:cancel', 'run.settled']); + const events = store.readRun(out.runId); + const stall = events.find((e) => e.type === 'liveness.stalled').data; + assert.equal(stall.windowMs, 60); + assert.ok(stall.idleMs >= 60, `idleMs ${stall.idleMs} is at least the window`); + assert.deepEqual(events.find((e) => e.type === 'liveness.recovery').data, { action: 'cancel', attempt: 1 }); + assert.equal(settledOf(store, out.runId)[0].data.failure.kind, 'stuck'); + assert.equal(stalled.length, 1); + assert.equal(stalled[0].runId, out.runId); + assert.equal(stalled[0].attempt, 1); +})); + +test('a hung adapter that IGNORES abort still settles stuck, records kill, and kills its child', () => withStore(async (store) => { + const child = fakeChild(); + const adapter = { + name: 'hang-ignore', + spawn(prompt, ctx) { + ctx.onSpawn({ model: null, effort: null, at: 'x' }, child); + return new Promise(() => {}); // never settles, never looks at the signal + }, + }; + const started = Date.now(); + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + const elapsed = Date.now() - started; + assert.equal(out.outcome, 'stuck'); + assert.equal(out.failure.kind, 'stuck'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.match(out.failure.detail, /did not settle within 60ms of cancellation/); + assert.deepEqual(child.kills, ['SIGKILL']); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:kill', 'run.settled']); + assert.ok(elapsed < 1000, `settled in ${elapsed}ms, not left running`); +})); + +test('an ignored cancellation is not retried even when the kind is listed', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'hang-ignore', spawn() { calls += 1; return new Promise(() => {}); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 2 } } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(calls, 1, 'a second attempt could race the first, which may still be running'); +})); + +test('steady trace progress is never called stuck', () => withStore(async (store) => { + const adapter = { + name: 'steady', + async spawn(prompt, ctx) { + for (let i = 0; i < 15; i += 1) { // 300ms of work against a 60ms window + await delay(20); + ctx.trace.emit('agent.thought_chunk', { chars: 1 }); + } + return { text: 'done' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.ok(!typesOf(store, out.runId).some((t) => t.startsWith('liveness') || t.startsWith('recovery'))); +})); + +test('child stdout/stderr bytes count as progress, and the caller\'s onSpawn still receives every argument', () => withStore(async (store) => { + const seen = []; + const adapter = { + name: 'bytes', + async spawn(prompt, ctx) { + const child = fakeChild(); + observeSpawn(child, ctx.onSpawn, { model: 'm' }); + child.emit('spawn'); + for (let i = 0; i < 15; i += 1) { + await delay(20); + (i % 2 ? child.stdout : child.stderr).emit('data', Buffer.from('.')); + } + return { text: 'done' }; + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { onSpawn: (...args) => seen.push(args) }, policy: FAST }); + assert.equal(out.outcome, 'ok'); + assert.equal(seen.length, 1); + assert.equal(seen[0][0].model, 'm'); + assert.equal(typeof seen[0][1].kill, 'function', 'observeSpawn passes the child as the second argument'); + assert.ok(!typesOf(store, out.runId).includes('liveness.stalled')); +})); + +test('progress that stops part-way is caught, and idleMs is measured from the last signal', () => withStore(async (store) => { + const adapter = { + name: 'stops', + spawn(prompt, ctx) { + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => reject(ctx.signal.reason)); + (async () => { for (let i = 0; i < 5; i += 1) { await delay(20); ctx.trace.emit('agent.thought_chunk', { chars: 1 }); } })(); + }); + }, + }; + const started = Date.now(); + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'stuck'); + const stall = store.readRun(out.runId).find((e) => e.type === 'liveness.stalled').data; + assert.ok(stall.idleMs >= 60 && stall.idleMs < 200, `idleMs ${stall.idleMs}`); + assert.ok(Date.now() - started >= 160, 'the window restarted on each signal'); +})); + +test('a refusal is never retried, even when listed in retry.on', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'refuses', async spawn() { calls += 1; throw runFailure('refusal', 'content policy'); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 3, on: ['refusal', 'stuck'] } } }); + assert.equal(calls, 1); + assert.equal(out.outcome, 'failed'); + assert.equal(out.attempts, 1); + assert.equal(out.failure.kind, 'refusal'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); + assert.deepEqual(settledOf(store, out.runId)[0].data.failure, { kind: 'refusal', detail: 'content policy' }); +})); + +test('stuck is retried once on the same runId, carrying the session, then succeeds', () => withStore(async (store) => { + const calls = []; + const adapter = { + name: 'flaky', + spawn(prompt, ctx) { + calls.push({ sessionId: ctx.sessionId }); + if (calls.length === 2) return Promise.resolve({ text: 'second time', newSessionId: 'sess-1' }); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => { + const failure = ctx.signal.reason; + failure.sessionId = 'sess-1'; // the adapter knew its session when it failed + reject(failure); + }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 1 } } }); + assert.equal(out.outcome, 'ok'); + assert.equal(out.attempts, 2); + assert.equal(out.result.text, 'second time'); + assert.deepEqual(calls, [{ sessionId: undefined }, { sessionId: 'sess-1' }]); + assert.deepEqual(typesOf(store, out.runId), [ + 'run.started', 'liveness.stalled', 'recovery:cancel', 'recovery:retry', 'run.started', 'run.settled']); + const starts = store.readRun(out.runId).filter((e) => e.type === 'run.started').map((e) => e.data); + assert.deepEqual(starts, [{ adapter: 'flaky', attempt: 1 }, { adapter: 'flaky', sessionId: 'sess-1', attempt: 2 }]); + assert.equal(settledOf(store, out.runId)[0].data.attempts, 2); + assert.equal(store.listRuns().length, 1, 'a retry is the same run, not a new one'); +})); + +test('retry is off by default: a stuck run is attempted once', () => withStore(async (store) => { + const calls = []; + const out = await superviseRun({ adapter: hangsHonouringAbort(calls), prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000 } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(calls.length, 1); +})); + +test('retry stops at max and settles with the last failure', () => withStore(async (store) => { + const calls = []; + const out = await superviseRun({ adapter: hangsHonouringAbort(calls), prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000, retry: { max: 2 } } }); + assert.equal(calls.length, 3); + assert.equal(out.attempts, 3); + assert.equal(out.outcome, 'stuck'); + assert.equal(settledOf(store, out.runId).length, 1); +})); + +test('a kind left out of retry.on is not retried', () => withStore(async (store) => { + let calls = 0; + const adapter = { name: 'exits', async spawn() { calls += 1; throw runFailure('exited', 'code 1'); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, retry: { max: 2, on: ['stuck'] } } }); + assert.equal(calls, 1); + assert.equal(out.outcome, 'failed'); +})); + +test('the wall-clock timeout cancels a busy run with kind timeout, not stuck', () => withStore(async (store) => { + const adapter = { + name: 'busy', + spawn(prompt, ctx) { + return new Promise((_, reject) => { + const tick = setInterval(() => ctx.trace.emit('agent.thought_chunk', { chars: 1 }), 10); + ctx.signal.addEventListener('abort', () => { clearInterval(tick); reject(ctx.signal.reason); }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { stallMs: 1000, cancelGraceMs: 1000, timeoutMs: 120 } }); + assert.equal(out.outcome, 'failed'); + assert.equal(out.failure.kind, 'timeout'); + const types = typesOf(store, out.runId); + assert.ok(!types.includes('liveness.stalled')); + assert.deepEqual(types.filter((t) => !t.startsWith('agent.')), ['run.started', 'recovery:cancel', 'run.settled']); + assert.equal(settledOf(store, out.runId)[0].data.failure.kind, 'timeout'); +})); + +test('a timeout the adapter ignores still settles as timeout', () => withStore(async (store) => { + const adapter = { name: 'deaf', spawn: () => new Promise(() => {}) }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { stallMs: 0, cancelGraceMs: 40, timeoutMs: 60 } }); + assert.equal(out.failure.kind, 'timeout'); + assert.equal(out.failure.adapterIgnoredCancel, true); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'recovery:cancel', 'recovery:kill', 'run.settled']); +})); + +test('a caller\'s signal cancels the run as cancelled, and a pre-aborted signal never spawns', () => withStore(async (store) => { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 30); + const out = await superviseRun({ adapter: hangsHonouringAbort(), prompt: 'p', store, ctx: { signal: controller.signal }, policy: { ...FAST, stallMs: 1000, retry: { max: 3 } } }); + assert.equal(out.outcome, 'cancelled'); + assert.equal(out.failure.kind, 'cancelled'); + assert.equal(out.attempts, 1); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); + + let spawned = 0; + const out2 = await superviseRun({ adapter: { name: 'x', spawn: () => { spawned += 1; return {}; } }, prompt: 'p', store, ctx: { signal: AbortSignal.abort() }, policy: FAST }); + assert.equal(out2.outcome, 'cancelled'); + assert.equal(spawned, 0); +})); + +test('a bare Error settles failed with no kind, scrubbed, and does not throw', () => withStore(async (store) => { + const key = 'sk-or-v1-abcdefabcdefabcdef'; + const adapter = { name: 'bare', providerKeyEnv: 'OPENROUTER_API_KEY', spawn() { throw new Error(`exploded with ${key}`); } }; + const out = await superviseRun({ adapter, prompt: 'p', store, ctx: { env: { OPENROUTER_API_KEY: key } }, policy: FAST }); + assert.equal(out.outcome, 'failed'); + const { failure } = settledOf(store, out.runId)[0].data; + assert.equal(failure.kind, null); + assert.doesNotMatch(failure.detail, /abcdefabcdef/); + assert.match(failure.detail, /exploded with \[redacted\]/); +})); + +test('an adapter cannot write the supervisor\'s run.* events', () => withStore(async (store) => { + const adapter = { + name: 'forger', + async spawn(prompt, ctx) { + assert.equal(ctx.trace.emit('run.settled', { outcome: 'ok', durationMs: 0, attempts: 1 }), null); + assert.equal(ctx.trace.emit('run.started', { attempt: 9 }), null); + throw runFailure('exited', 'code 2'); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: FAST }); + assert.equal(out.outcome, 'failed'); + assert.deepEqual(typesOf(store, out.runId), ['run.started', 'run.settled']); +})); + +test('exactly one run.settled on every path, and a late adapter answer writes nothing', () => withStore(async (store) => { + const lateHook = []; + const late = { + name: 'late', + spawn(prompt, ctx) { + // Ignores the abort, then answers after the supervisor gave up on it. + return new Promise((resolve) => setTimeout(() => { + lateHook.push(ctx.trace.emit('agent.message_chunk', { text: 'too late' })); + resolve({ text: 'too late' }); + }, 250)); + }, + }; + const paths = [ + { adapter: { name: 'ok', spawn: async () => ({ text: 'x' }) }, outcome: 'ok' }, + { adapter: hangsHonouringAbort(), outcome: 'stuck' }, + { adapter: { name: 'deaf', spawn: () => new Promise(() => {}) }, outcome: 'stuck' }, + { adapter: late, outcome: 'stuck' }, + { adapter: { name: 'refuse', spawn: async () => { throw runFailure('refusal', 'no'); } }, outcome: 'failed', policy: { retry: { max: 2, on: ['refusal'] } } }, + { adapter: hangsHonouringAbort(), outcome: 'stuck', policy: { retry: { max: 1 } } }, + { adapter: { name: 'deaf', spawn: () => new Promise(() => {}) }, outcome: 'failed', policy: { stallMs: 0, timeoutMs: 50 } }, + { adapter: { name: 'bare', spawn: () => { throw new Error('boom'); } }, outcome: 'failed' }, + ]; + const runIds = []; + for (const path of paths) { + const out = await superviseRun({ adapter: path.adapter, prompt: 'p', store, policy: { ...FAST, ...path.policy } }); + assert.equal(out.outcome, path.outcome, path.adapter.name); + runIds.push(out.runId); + } + await delay(300); // let the late adapter answer + assert.deepEqual(lateHook, [null], 'the late trace write was dropped'); + for (const runId of runIds) { + assert.equal(settledOf(store, runId).length, 1, runId); + assert.equal(store.readRun(runId).at(-1).type, 'run.settled', 'nothing follows the settle'); + } + assert.ok(store.listRuns({ limit: 100 }).every((row) => row.outcome !== 'running')); +})); + +test('an adapter that answers after the supervisor aborted it still settles with the abort reason', () => withStore(async (store) => { + const adapter = { + name: 'answers-anyway', + spawn(prompt, ctx) { + return new Promise((resolve) => { + ctx.signal.addEventListener('abort', () => resolve({ text: 'a reply nobody waited for' })); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, cancelGraceMs: 1000 } }); + assert.equal(out.outcome, 'stuck'); + assert.equal(out.result, undefined); + assert.equal(settledOf(store, out.runId)[0].data.outcome, 'stuck'); +})); + +test('a cancelled attempt\'s late trace writes never reach the next attempt', () => withStore(async (store) => { + let calls = 0; + const adapter = { + name: 'stray', + spawn(prompt, ctx) { + calls += 1; + if (calls === 2) return delay(100).then(() => ({ text: 'ok' })); + return new Promise((_, reject) => { + ctx.signal.addEventListener('abort', () => { + reject(ctx.signal.reason); + setTimeout(() => ctx.trace.emit('agent.message_chunk', { text: 'stale' }), 30); + }); + }); + }, + }; + const out = await superviseRun({ adapter, prompt: 'p', store, policy: { ...FAST, stallMs: 200, retry: { max: 1 } } }); + assert.equal(out.outcome, 'ok'); + assert.equal(out.attempts, 2); + assert.ok(!store.readRun(out.runId).some((e) => e.data?.text === 'stale')); +})); From bbfc39a388d30e1391eb96225883eb5976903964 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:30:56 -0700 Subject: [PATCH 18/20] feat(seats): record-v1 adds agent.retry, and states the hook attribution rule agent.retry records an agent reporting that it is retrying its provider: {state: scheduled|started, attempt?, nextRetryAt?, status?, errorType?}. Each field is absent when unknown, never null. status is an integer HTTP code from 100 to 599. errorType is a closed enum: the provider error types opencode 2.1.4's error formatter emits, read from the installed 2.1.4 binary, plus 'other'. A writer maps anything else to 'other', and the validator refuses any value outside the enum and any field beyond the five, so neither the agent's error message nor the provider's response body can be recorded. docs/record-v1.md documents the kind, and states that a permission hook answer naming nobody gets the reject default, recorded as policy/policy, with a decider marked recordsPermissions exempt. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --- packages/seats/docs/record-v1.md | 33 +++++++++++-- packages/seats/src/record/format.js | 33 +++++++++++++ packages/seats/src/trace/events.js | 1 + packages/seats/test/record-format.test.js | 58 +++++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) diff --git a/packages/seats/docs/record-v1.md b/packages/seats/docs/record-v1.md index 26ab8f459f..bc1a392764 100644 --- a/packages/seats/docs/record-v1.md +++ b/packages/seats/docs/record-v1.md @@ -217,6 +217,12 @@ it does not prove the process lost access. That happens at `run.settled`. - A rule-decided call is `"policy"`. So is the default given when no answer came in time, or when the answer did not name a user and an owner role: such an answer approves nothing. + - A permission hook's answer is held to the same rule. An answer that does + not name who decided (`decidedBy` a user id, plus an owner + `decidedByRole`) gets the reject default, recorded as + `decidedBy`/`decidedByRole` `policy`/`policy` with `approval: failed`. A + decider marked `recordsPermissions` (the valet's policy) writes and + attributes its own entries, so it is exempt. - It is `"policy"` exactly when `decidedByRole` is `"policy"`. - `decidedByRole` is one of: - `credential-owner`: the owner of the connection the call would use. This @@ -401,13 +407,31 @@ documented in `src/trace/events.js`. | kind | default actor | |---|---| | `turn.prompt_sent` | `system` | -| `agent.message_chunk`, `agent.thought_chunk`, `plan`, `usage` | run | +| `agent.message_chunk`, `agent.thought_chunk`, `agent.retry`, `plan`, `usage` | run | | `liveness.stalled`, `liveness.recovery`, `process.exited` | `system` | `usage` may also carry `contextTokens` (how much of the agent's context window is in use) and `contextSize` (the window's size). Each is an integer `>= 0`, or `null` when unknown; a writer refuses any other value. +`agent.retry` is the agent reporting that it is retrying its provider: +`{state, attempt?, nextRetryAt?, status?, errorType?}`. +- `state` is `scheduled` (a retry waits until `nextRetryAt`) or `started` + (the waited-for attempt went out). +- `attempt` is an integer `>= 1`, and `nextRetryAt` an RFC 3339 UTC + timestamp. +- `status` is an integer HTTP status, `100` to `599`. +- `errorType` is one of `provider.auth`, `provider.content-filter`, + `provider.free-limit`, `provider.internal`, `provider.invalid-output`, + `provider.invalid-request`, `provider.no-route`, `provider.quota`, + `provider.rate-limit`, `provider.timeout`, `provider.transport`, + `provider.unknown`, `provider.unsupported-operation` or `other`. These are + the provider error types opencode 2.1.4 reports on a retry; a writer maps + any other type to `other`. +- A field that is not known is absent, never `null`. The agent's error + message and the provider's response body are never recorded, and a writer + refuses any field not listed here. + ### Mapping from the trace vocabulary Emitters keep calling `emit(type, data)` with `EVENT_TYPES` names, and the @@ -469,9 +493,10 @@ What's new or changed in `EVENT_TYPES`: lines still verify. A later revision, also at `v: 1`, added the optional `actor.onBehalfOf`, - the kinds `session.opened`, `sandbox.applied`, `egress.allowed` and - `egress.denied`, `contextTokens` and `contextSize` on `usage`, and the - `pod-member` value of `decidedByRole`. Lines without them are unchanged. + the kinds `session.opened`, `sandbox.applied`, `egress.allowed`, + `egress.denied` and `agent.retry`, `contextTokens` and `contextSize` on + `usage`, and the `pod-member` value of `decidedByRole`. Lines without them + are unchanged. 4. **One writer per file.** A single process appends lines in `seq` order. ## 4. What the hash chain proves, and what it doesn't diff --git a/packages/seats/src/record/format.js b/packages/seats/src/record/format.js index 53027db99c..2612d8a91c 100644 --- a/packages/seats/src/record/format.js +++ b/packages/seats/src/record/format.js @@ -35,6 +35,28 @@ export const EGRESS_DENIED_REASONS = Object.freeze([ const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); export const SANDBOXES = Object.freeze(['seatbelt', 'bwrap', 'none']); export const SESSION_METHODS = Object.freeze(['new', 'resume', 'load']); +export const RETRY_STATES = Object.freeze(['scheduled', 'started']); +/** + * agent.retry.errorType. The provider error types opencode 2.1.4 reports on a + * retry, read from its error formatter, plus `other` for any type outside + * them. A closed set, so no agent text can ride in this field. + */ +export const RETRY_ERROR_TYPES = Object.freeze([ + 'provider.auth', + 'provider.content-filter', + 'provider.free-limit', + 'provider.internal', + 'provider.invalid-output', + 'provider.invalid-request', + 'provider.no-route', + 'provider.quota', + 'provider.rate-limit', + 'provider.timeout', + 'provider.transport', + 'provider.unknown', + 'provider.unsupported-operation', + 'other', +]); // Why a requested session was not reopened: the two forms an ACP adapter // writes. The code is the JSON-RPC error's, an integer by JSON-RPC 2.0. const SESSION_FALLBACK = /^(?:agent supports neither session\/resume nor session\/load|session\/(?:resume|load) answered error -?\d{1,16})$/; @@ -70,6 +92,7 @@ export const KINDS = Object.freeze({ 'turn.prompt_sent': Object.freeze({ group: 'activity', actor: 'system' }), 'agent.message_chunk': Object.freeze({ group: 'activity', actor: 'run' }), 'agent.thought_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.retry': Object.freeze({ group: 'activity', actor: 'run' }), plan: Object.freeze({ group: 'activity', actor: 'run' }), usage: Object.freeze({ group: 'activity', actor: 'run' }), 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), @@ -378,6 +401,16 @@ const CHECKS = { } return data; }, + 'agent.retry': (data) => { + refuseUnknownKeys(data, ['state', 'attempt', 'nextRetryAt', 'status', 'errorType'], 'agent.retry.'); + if (!RETRY_STATES.includes(data.state)) fail(`agent.retry.state must be one of ${RETRY_STATES.join(', ')}`); + // Every other field is absent when unknown, never null. + if ('attempt' in data && !(Number.isInteger(data.attempt) && data.attempt >= 1)) fail('agent.retry.attempt must be an integer >= 1, or absent'); + if ('nextRetryAt' in data && !isTimestamp(data.nextRetryAt)) fail('agent.retry.nextRetryAt must be an RFC 3339 UTC timestamp, or absent'); + if ('status' in data && !(Number.isInteger(data.status) && data.status >= 100 && data.status <= 599)) fail('agent.retry.status must be an integer HTTP status (100-599), or absent'); + if ('errorType' in data && !RETRY_ERROR_TYPES.includes(data.errorType)) fail(`agent.retry.errorType must be one of ${RETRY_ERROR_TYPES.join(', ')}, or absent`); + return data; + }, usage: (data) => { for (const key of ['contextTokens', 'contextSize']) { const value = data[key]; diff --git a/packages/seats/src/trace/events.js b/packages/seats/src/trace/events.js index 0ababca921..229a7ca7f9 100644 --- a/packages/seats/src/trace/events.js +++ b/packages/seats/src/trace/events.js @@ -14,6 +14,7 @@ export const EVENT_TYPES = Object.freeze([ 'turn.prompt_sent', // { chars } 'agent.message_chunk', // { text } 'agent.thought_chunk', // { chars } thoughts are counted, not stored + 'agent.retry', // { state: 'scheduled'|'started', attempt?, nextRetryAt?, status?, errorType? } the agent retrying its provider; never the provider's text 'tool.call', // { toolCallId, title, kind?, status?, name?, surface?, argsDigest? } 'tool.update', // { toolCallId, status?, title? } 'plan', // { entries: [{content, status}] } diff --git a/packages/seats/test/record-format.test.js b/packages/seats/test/record-format.test.js index 040c7b2236..beebd91b24 100644 --- a/packages/seats/test/record-format.test.js +++ b/packages/seats/test/record-format.test.js @@ -1,8 +1,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { appendEntry, argsDigestOf, canonicalLine, hashLine, readRecord, verifyRecord, EGRESS_DENIED_REASONS, KINDS, ON_BEHALF_ROLES, SANDBOXES, SESSION_METHODS, RECORD_VERSION, RECORD_SCHEMA, toolSurfaceOf, + RETRY_ERROR_TYPES, RETRY_STATES, } from '../src/record/format.js'; const RUN = 'run-fmt-0001'; @@ -499,3 +502,58 @@ test('usage: contextTokens and contextSize are integers >= 0, or null when unkno for (const value of [-1, 1.5, '41000', Number.NaN]) assert.throws(() => at({ [key]: value }), new RegExp(`usage.${key}`), `${key} ${value}`); } }); + +test('agent.retry: state, attempt, nextRetryAt, status and errorType, each absent when unknown', () => { + const at = (data) => JSON.parse(appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT })); + const full = at({ state: 'scheduled', attempt: 2, nextRetryAt: '2026-10-25T13:00:02.000Z', status: 503, errorType: 'provider.internal' }); + assert.equal(full.kind, 'agent.retry'); + assert.equal(full.actor.id, AGENT.id); + at({ state: 'started', attempt: 2 }); + at({ state: 'started' }); + for (const errorType of RETRY_ERROR_TYPES) at({ state: 'scheduled', errorType }); + assert.ok(RETRY_ERROR_TYPES.includes('other')); + for (const state of [undefined, 'waiting', 'SCHEDULED']) assert.throws(() => at({ state }), /agent.retry.state/, String(state)); + for (const attempt of [0, -1, 1.5, '2', null]) assert.throws(() => at({ state: 'started', attempt }), /agent.retry.attempt/, String(attempt)); + for (const nextRetryAt of ['soon', '2026-10-25 13:00:02', 12, null]) assert.throws(() => at({ state: 'scheduled', nextRetryAt }), /agent.retry.nextRetryAt/, String(nextRetryAt)); + // Provider text has no field to ride in. + for (const key of ['message', 'body', 'response', 'error']) { + assert.throws(() => at({ state: 'scheduled', [key]: 'upstream said no' }), new RegExp(`unknown field agent.retry.${key}`)); + } + assert.deepEqual(RETRY_STATES, ['scheduled', 'started']); +}); + +test('agent.retry.errorType is a closed enum: any other value is refused', () => { + // opencode 2.1.4's provider error types, read from the installed package, + // plus `other`. Changing this set is a record-v1 change, so it is pinned here. + assert.deepEqual([...RETRY_ERROR_TYPES], [ + 'provider.auth', 'provider.content-filter', 'provider.free-limit', 'provider.internal', + 'provider.invalid-output', 'provider.invalid-request', 'provider.no-route', 'provider.quota', + 'provider.rate-limit', 'provider.timeout', 'provider.transport', 'provider.unknown', + 'provider.unsupported-operation', 'other', + ]); + assert.ok(Object.isFrozen(RETRY_ERROR_TYPES)); + const at = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT }); + for (const errorType of ['provider.made-up', 'Provider.Internal', 'provider.internal ', 'server_error', 'upstream said sk-live-1234', '', null, 42, ['provider.internal']]) { + assert.throws(() => at({ state: 'scheduled', errorType }), /agent.retry.errorType must be one of/, JSON.stringify(errorType)); + } +}); + +test('agent.retry.status is an integer HTTP status from 100 to 599, or absent', () => { + const at = (data) => appendEntry(null, { ...ENV, ts: ts(), kind: 'agent.retry', data }, { runActor: AGENT }); + at({ state: 'scheduled', status: 100 }); + at({ state: 'scheduled', status: 599 }); + at({ state: 'scheduled' }); + for (const status of [99, 600, 0, -503, 503.5, '503', null, Number.NaN]) { + assert.throws(() => at({ state: 'scheduled', status }), /agent.retry.status/, String(status)); + } +}); + +test('record-v1.md states the hook attribution rule and lists every agent.retry error type', () => { + const doc = readFileSync(fileURLToPath(new URL('../docs/record-v1.md', import.meta.url)), 'utf8').replace(/\s+/g, ' '); + assert.match(doc, /A permission hook's answer is held to the same rule\. An answer that does not name who decided \(`decidedBy` a user id, plus an owner `decidedByRole`\) gets the reject default, recorded as `decidedBy`\/`decidedByRole` `policy`\/`policy`/); + assert.match(doc, /A decider marked `recordsPermissions` \(the valet's policy\) writes and attributes its own entries, so it is exempt\./); + const retry = doc.slice(doc.indexOf('`agent.retry` is the agent reporting'), doc.indexOf('### Mapping from the trace vocabulary')); + assert.ok(retry.length > 0, 'the agent.retry paragraph is present'); + for (const type of RETRY_ERROR_TYPES) assert.ok(retry.includes(`\`${type}\``), `the doc lists ${type}`); + for (const [, type] of retry.matchAll(/`(provider\.[a-z-]+)`/g)) assert.ok(RETRY_ERROR_TYPES.includes(type), `the doc lists only the set: ${type}`); +}); From c226bcaedfd6d06bfdca423b6053dce65b7784a1 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:38:17 -0400 Subject: [PATCH 19/20] fix(seats): a supervised kill reaches the CLI's whole process tree; a handled SIGTERM keeps the turn (#2726) * fix(seats): the supervisor kills a stalled turn's whole process group killChildren() sent SIGKILL to the direct child only, so an agent CLI's own children (MCP servers started through npx, the native binary behind codex's node launcher) survived the kill and were orphaned on the shared host. - process-group.js: spawnInOwnGroup starts a child detached and marks it (LEADS_PROCESS_GROUP). killProcessTree signals -pid only for a marked child that has not been reaped, and otherwise falls back to child.kill. Leading a group is never inferred. - The supervisor passes ctx.ownProcessGroup = true and kills through killProcessTree. - The claude and codex adapters spawn through spawnInOwnGroup and ask for a group only when ctx.ownProcessGroup === true. Unsupervised, the options object reaches spawn untouched, so live seats spawn exactly as before. - Wrapper exit: a detached group is outside the seat's launchd group. While a supervised group is live, the wrapper kills every tracked group on SIGTERM, SIGINT, SIGHUP and exit. When nothing else handles the signal, it re-raises the signal so the wrapper still dies by it. Tests use real processes: a grandchild left by `sleep 300 &` is gone after a stall kill and after the wrapper is signalled or exits, a child that does not lead a group gets child.kill with no negative pid, and the unsupervised claude/codex spawn carries no detached option. Mutation check: reverting killChildren to child.kill alone fails the grandchild test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(seats): walk a killed child's descendants; a host-handled SIGTERM keeps the turn A group kill misses descendants that moved to a group of their own. Live codex seats put every MCP server and tool command in a group of its own, and claude's Bash tool shells do the same. - Descendant walk: before any signal, snapshot `ps -A -o pid=,ppid=,lstart=`. Then signal the child's descendants deepest first, re-reading each pid right before its signal. A pid is killed only if its ppid and start time still match the snapshot, so a pid reused by another process on the host is left alone. The walk runs before the group kill, so each pid's parent is still alive when its ppid is checked. Nothing loosens the ppid match. - SIGTERM/SIGINT/SIGHUP: when a host handles the signal itself, the registry no longer kills groups, so the in-flight turn survives. With no host handler, it still kills them and re-raises. - armSupervisedGroupDeadline(ms): the host kills the groups by a deadline it sets before launchd's SIGKILL, which no handler can catch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(cli): agent run kills supervised groups before the seat's ExitTimeOut On SIGTERM, `agent run` still stops polling and finishes the in-flight turn. It now also arms a supervised-group kill a margin before ExitTimeOut, which it reads (only) from ~/Library/LaunchAgents/me.commonly.seat-<name>.plist. An unset, zero or unreadable value means launchd's 20 s default. The margin is min(5 s, a quarter), so the default kill lands at 15 s. Its exit path calls killSupervisedGroups() for groups still live once the turn ends. With no supervised group live, both calls do nothing, so today's seats are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * test(seats): wait for the unrelated process's spawn event before it can fire unseen The pid-reuse test awaited `once(unrelated, 'spawn')` only after reading the grandchildren's pids. By then the event had already fired, so the test hung until node's test timeout. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu * fix(cli): arm the supervised-group deadline only for a seat with a plist The daemon spawns some seats, and those have no plist. The daemon stops a seat with SIGTERM and never sends SIGKILL, so nothing there needs launchd's 20 s fallback. Arming it meant a reassignment or config change cut an in-flight turn's tools at 15 s. - readSeatPlist reports whether the seat's plist exists. With no plist, SIGTERM arms nothing and the exit path alone kills the supervised groups. A plist that exists but has no ExitTimeOut, or can't be read, keeps launchd's 20 s default, because launchd will SIGKILL that seat. - process-group.js header: document the walk's two known gaps. A descendant reparented to 1 mid-walk is skipped. A process started after the snapshot is missed, and so is one that calls setsid after it. Tests use real processes. With a plist, the groups die before ExitTimeOut while the turn is still running. With no plist, the turn runs past 15 s and the groups die on exit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- cli/__tests__/adapters.process-group.test.mjs | 67 +++++ cli/__tests__/seat-stop.process.test.mjs | 136 +++++++++ cli/__tests__/seat-stop.test.mjs | 128 +++++++++ cli/src/commands/agent.js | 9 +- cli/src/lib/adapters/claude.js | 11 +- cli/src/lib/adapters/codex.js | 7 +- cli/src/lib/seat-stop.js | 96 +++++++ packages/seats/src/index.js | 5 + packages/seats/src/process-group.js | 261 ++++++++++++++++++ packages/seats/src/supervisor.js | 18 +- packages/seats/test-support/processes.js | 30 ++ .../seats/test/supervisor-group-kill.test.js | 218 +++++++++++++++ .../test/supervisor-tree-and-deadline.test.js | 225 +++++++++++++++ 13 files changed, 1201 insertions(+), 10 deletions(-) create mode 100644 cli/__tests__/adapters.process-group.test.mjs create mode 100644 cli/__tests__/seat-stop.process.test.mjs create mode 100644 cli/__tests__/seat-stop.test.mjs create mode 100644 cli/src/lib/seat-stop.js create mode 100644 packages/seats/src/process-group.js create mode 100644 packages/seats/test-support/processes.js create mode 100644 packages/seats/test/supervisor-group-kill.test.js create mode 100644 packages/seats/test/supervisor-tree-and-deadline.test.js diff --git a/cli/__tests__/adapters.process-group.test.mjs b/cli/__tests__/adapters.process-group.test.mjs new file mode 100644 index 0000000000..715b5becd4 --- /dev/null +++ b/cli/__tests__/adapters.process-group.test.mjs @@ -0,0 +1,67 @@ +/** + * The claude and codex adapters start their child in its own process group + * only when the run supervisor asks (ctx.ownProcessGroup), because only the + * supervisor kills the group. Live seats run unsupervised, and their spawn + * options must stay exactly what they were. + */ +import { jest } from '@jest/globals'; +import { EventEmitter } from 'events'; + +await jest.unstable_mockModule('child_process', () => ({ + spawnSync: jest.fn(() => ({ status: 1, stdout: '', stderr: '' })), + spawn: jest.fn(), +})); + +const claude = (await import('../src/lib/adapters/claude.js')).default; +const codex = (await import('../src/lib/adapters/codex.js')).default; +const { LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP } = await import('@commonlyai/seats/process-group'); + +const makeSpawnImpl = () => { + const calls = []; + const impl = (cmd, args, opts) => { + const proc = new EventEmitter(); + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + proc.kill = jest.fn(); + calls.push({ cmd, args, opts, proc }); + setTimeout(() => { + proc.stdout.emit('data', Buffer.from('ok')); + proc.emit('close', 0); + }, 0); + return proc; + }; + return { impl, calls }; +}; + +const ADAPTERS = [['claude', claude], ['codex', codex]]; +const POSIX = process.platform !== 'win32'; + +describe.each(ADAPTERS)('%s adapter — process group', (name, adapter) => { + test('unsupervised: spawns with exactly cwd, env and stdio, no detached, no marker', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl }); + expect(calls).toHaveLength(1); + expect(Object.keys(calls[0].opts).sort()).toEqual(['cwd', 'env', 'stdio']); + expect(calls[0].opts.stdio).toEqual(['ignore', 'pipe', 'pipe']); + expect(calls[0].proc[LEADS_PROCESS_GROUP]).toBeUndefined(); + }); + + (POSIX ? test : test.skip)('supervised: spawns detached and marks the child as leading its group', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { + environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl, [OWN_PROCESS_GROUP]: true, + }); + expect(calls).toHaveLength(1); + expect(calls[0].opts.detached).toBe(true); + expect(Object.keys(calls[0].opts).sort()).toEqual(['cwd', 'detached', 'env', 'stdio']); + expect(calls[0].proc[LEADS_PROCESS_GROUP]).toBe(true); + }); + + test('a truthy non-true value does not opt in', async () => { + const { impl, calls } = makeSpawnImpl(); + await adapter.spawn('hi', { + environment: { model: 'test-model' }, sessionId: null, _spawnImpl: impl, [OWN_PROCESS_GROUP]: 'yes', + }); + expect(calls[0].opts).not.toHaveProperty('detached'); + }); +}); diff --git a/cli/__tests__/seat-stop.process.test.mjs b/cli/__tests__/seat-stop.process.test.mjs new file mode 100644 index 0000000000..d4433402f4 --- /dev/null +++ b/cli/__tests__/seat-stop.process.test.mjs @@ -0,0 +1,136 @@ +/** + * seat-stop with real processes, wired the way `agent run` wires it: SIGTERM + * calls requestStop, and the exit path calls onExit. The supervised child + * leads its own group and holds a grandchild (`sleep 300 &`), the shape of a + * CLI holding an MCP server. + * + * - A launchd seat (its plist exists): the groups die a margin before + * ExitTimeOut, while the turn is still running. + * - A daemon-spawned seat (no plist): nothing is armed, the turn runs past the + * 15 s a default deadline would have cut it at, and the groups die on exit. + */ +import { spawn } from 'child_process'; +import { once } from 'events'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; +import { fileURLToPath, pathToFileURL } from 'url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const SEAT_STOP = pathToFileURL(join(HERE, '..', 'src', 'lib', 'seat-stop.js')).href; +const SEATS = join(HERE, '..', '..', 'packages', 'seats', 'src'); +const seats = (file) => JSON.stringify(pathToFileURL(join(SEATS, file)).href); +const POSIX = process.platform !== 'win32'; + +const isRunning = (pid) => { + try { process.kill(pid, 0); } catch (error) { return error.code !== 'ESRCH'; } + try { return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return true; } +}; +const sleep = (ms) => new Promise((r) => { setTimeout(r, ms); }); +const waitFor = async (predicate, ms = 5000) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (predicate()) return true; + await sleep(25); + } + return predicate(); +}; +const reap = (pid) => { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }; + +const HOST = ({ home, pidFile, goFile }) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { createSeatStop } from ${JSON.stringify(SEAT_STOP)}; +import { superviseRun } from ${seats('supervisor.js')}; +import { createTraceStore } from ${seats('trace/store.js')}; +import { observeSpawn } from ${seats('runtime-observation.js')}; +import { spawnInOwnGroup, wantsOwnProcessGroup } from ${seats('process-group.js')}; +let stopping = false; +const seatStop = createSeatStop({ name: 'probe', stop: () => { stopping = true; }, home: ${JSON.stringify(home)} }); +process.on('SIGTERM', (signal) => { process.stdout.write('stopping deadline=' + seatStop.requestStop(signal) + '\\n'); }); +process.once('exit', seatStop.onExit); +const script = "sleep 300 & echo $! > '${pidFile}'; while [ ! -f '${goFile}' ]; do sleep 0.05; done; echo done; wait"; +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', script], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.stdout.on('data', (chunk) => { if (String(chunk).includes('done')) resolve({ text: 'done' }); }); + child.on('close', (code, signal) => reject(new Error('sh ended ' + (code ?? signal)))); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + }, 20); + }); +}) }; +const out = await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(join(home, 'trace'))} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +process.stdout.write('turn ' + out.outcome + '\\n'); +if (stopping) process.exit(0); +`; + +let home; +let state; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'seat-stop-proc-')); + state = null; +}); +afterEach(() => { + if (state) { + if (state.grandchild) reap(state.grandchild); + if (state.leader) reap(state.leader); + state.host.kill('SIGKILL'); + } + rmSync(home, { recursive: true, force: true }); +}); + +const startHost = async () => { + const pidFile = join(home, 'grandchild.pid'); + const goFile = join(home, 'go'); + const script = join(home, 'host.mjs'); + writeFileSync(script, HOST({ home, pidFile, goFile })); + const host = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + state = { host, goFile, out: '' }; + host.stdout.on('data', (chunk) => { state.out += chunk; }); + state.exited = once(host, 'exit'); + expect(await waitFor(() => /ready \d+/.test(state.out))).toBe(true); + state.leader = Number(state.out.match(/ready (\d+)/)[1]); + expect(await waitFor(() => existsSync(pidFile) && /^\d+\s*$/.test(readFileSync(pidFile, 'utf8')))).toBe(true); + state.grandchild = Number(readFileSync(pidFile, 'utf8')); + return state; +}; + +const describePosix = POSIX ? describe : describe.skip; + +describePosix('seat-stop — real processes', () => { + test('a seat with a plist: the groups die a margin before ExitTimeOut, mid-turn', async () => { + // ExitTimeOut 2 s → deadline 1.5 s. + mkdirSync(join(home, 'Library', 'LaunchAgents'), { recursive: true }); + writeFileSync(join(home, 'Library', 'LaunchAgents', 'me.commonly.seat-probe.plist'), + '<?xml version="1.0"?><plist version="1.0"><dict><key>ExitTimeOut</key><integer>2</integer></dict></plist>'); + await startHost(); + const sent = Date.now(); + state.host.kill('SIGTERM'); + expect(await waitFor(() => /stopping deadline=1500/.test(state.out))).toBe(true); + await sleep(750); + expect(isRunning(state.grandchild)).toBe(true); + expect(await waitFor(() => !isRunning(state.grandchild), 2000)).toBe(true); + expect(Date.now() - sent).toBeLessThan(2000); + expect(existsSync(state.goFile)).toBe(false); + }, 30000); + + test('a seat with no plist: nothing is armed, the turn runs past 15 s, and the groups die on exit', async () => { + await startHost(); + state.host.kill('SIGTERM'); + expect(await waitFor(() => /stopping deadline=null/.test(state.out))).toBe(true); + await sleep(16000); + expect(isRunning(state.grandchild)).toBe(true); + expect(isRunning(state.leader)).toBe(true); + writeFileSync(state.goFile, ''); + const [code] = await Promise.race([state.exited, sleep(5000).then(() => ['timeout'])]); + expect(state.out).toMatch(/turn ok/); + expect(code).toBe(0); + expect(await waitFor(() => !isRunning(state.grandchild))).toBe(true); + expect(await waitFor(() => !isRunning(state.leader))).toBe(true); + }, 40000); +}); diff --git a/cli/__tests__/seat-stop.test.mjs b/cli/__tests__/seat-stop.test.mjs new file mode 100644 index 0000000000..42934793ae --- /dev/null +++ b/cli/__tests__/seat-stop.test.mjs @@ -0,0 +1,128 @@ +/** + * seat-stop: SIGTERM keeps the turn, and arms a supervised-group kill a margin + * before the seat plist's ExitTimeOut (launchd's SIGKILL). The plist is only + * read. The process-level behaviour is covered with real processes in + * packages/seats/test/supervisor-tree-and-deadline.test.js. + */ +import { jest } from '@jest/globals'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { + DEFAULT_EXIT_TIMEOUT_S, createSeatStop, groupKillDeadlineMs, readExitTimeOut, readSeatPlist, seatPlistPath, +} from '../src/lib/seat-stop.js'; + +const plist = (body) => `<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>Label</key> + <string>me.commonly.seat-anvil</string> + <key>KeepAlive</key> + <true/> +${body} +</dict> +</plist> +`; + +let home; +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'seat-stop-')); + mkdirSync(join(home, 'Library', 'LaunchAgents'), { recursive: true }); +}); +afterEach(() => { rmSync(home, { recursive: true, force: true }); }); + +const writePlist = (name, body) => { + const path = seatPlistPath(name, home); + writeFileSync(path, plist(body)); + return path; +}; + +describe('seat-stop — ExitTimeOut', () => { + test('the plist path is the seat launch agent', () => { + expect(seatPlistPath('anvil', home)).toBe(join(home, 'Library', 'LaunchAgents', 'me.commonly.seat-anvil.plist')); + expect(seatPlistPath('../x', home)).toBeNull(); + expect(seatPlistPath('a/b', home)).toBeNull(); + }); + + test('reads ExitTimeOut from an XML plist, and only reads it', () => { + const path = writePlist('anvil', ' <key>ExitTimeOut</key>\n <integer>40</integer>'); + const before = readFileSync(path, 'utf8'); + expect(readExitTimeOut(path)).toBe(40); + expect(readFileSync(path, 'utf8')).toBe(before); + }); + + test('unset, zero, missing file and no path all read as unset', () => { + expect(readExitTimeOut(writePlist('a', ''))).toBeNull(); + expect(readExitTimeOut(writePlist('b', ' <key>ExitTimeOut</key>\n <integer>0</integer>'))).toBeNull(); + expect(readExitTimeOut(join(home, 'nope.plist'))).toBeNull(); + expect(readExitTimeOut(null)).toBeNull(); + }); + + test('a binary plist is read through plutil -extract', () => { + const run = jest.fn(() => ({ status: 0, stdout: '30\n' })); + const value = readExitTimeOut('/x.plist', { readFile: () => Buffer.from('bplist00...'), run }); + expect(value).toBe(30); + expect(run).toHaveBeenCalledWith('plutil', ['-extract', 'ExitTimeOut', 'raw', '-o', '-', '/x.plist'], expect.any(Object)); + }); + + test('the deadline is ExitTimeOut less min(5 s, a quarter); unset means 20 s', () => { + expect(DEFAULT_EXIT_TIMEOUT_S).toBe(20); + expect(groupKillDeadlineMs(null)).toBe(15000); + expect(groupKillDeadlineMs(40)).toBe(35000); + expect(groupKillDeadlineMs(4)).toBe(3000); + for (const s of [1, 4, 20, 40, 120]) expect(groupKillDeadlineMs(s)).toBeLessThan(s * 1000); + }); +}); + +describe('seat-stop — requestStop', () => { + const make = (name = 'anvil') => { + const stop = jest.fn(); + const arm = jest.fn(); + const kill = jest.fn(); + return { stop, arm, kill, seatStop: createSeatStop({ name, stop, home, deps: { arm, kill } }) }; + }; + + test('SIGTERM stops the loop and arms the group kill before ExitTimeOut', () => { + writePlist('anvil', ' <key>ExitTimeOut</key>\n <integer>8</integer>'); + const { stop, arm, kill, seatStop } = make(); + expect(seatStop.requestStop('SIGTERM')).toBe(6000); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).toHaveBeenCalledWith(6000); + expect(kill).not.toHaveBeenCalled(); + }); + + test('SIGTERM on a seat with no plist (daemon-spawned) arms nothing: the exit path alone kills', () => { + const { stop, arm, seatStop } = make('daemon-seat'); + expect(seatStop.requestStop('SIGTERM')).toBeNull(); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).not.toHaveBeenCalled(); + }); + + test('SIGTERM on a seat whose plist has no ExitTimeOut uses launchd\'s 20 s default', () => { + writePlist('anvil', ''); + const { arm, seatStop } = make(); + expect(seatStop.requestStop('SIGTERM')).toBe(15000); + expect(arm).toHaveBeenCalledWith(15000); + }); + + test('a plist that exists but cannot be read still counts as present, at the default', () => { + expect(readSeatPlist('/p.plist', { exists: () => true, readFile: () => { throw new Error('EACCES'); } })) + .toEqual({ present: true, exitTimeOut: null }); + expect(readSeatPlist(join(home, 'nope.plist'))).toEqual({ present: false, exitTimeOut: null }); + expect(readSeatPlist(null)).toEqual({ present: false, exitTimeOut: null }); + }); + + test('SIGINT stops the loop and arms nothing', () => { + const { stop, arm, seatStop } = make(); + expect(seatStop.requestStop('SIGINT')).toBeNull(); + expect(stop).toHaveBeenCalledTimes(1); + expect(arm).not.toHaveBeenCalled(); + }); + + test('the exit path kills the supervised groups', () => { + const { kill, seatStop } = make(); + seatStop.onExit(); + expect(kill).toHaveBeenCalledTimes(1); + }); +}); diff --git a/cli/src/commands/agent.js b/cli/src/commands/agent.js index fcd038aff7..b2e95554e4 100644 --- a/cli/src/commands/agent.js +++ b/cli/src/commands/agent.js @@ -83,6 +83,7 @@ import { detectBwrap } from '../lib/sandbox/bwrap.js'; import { resolvePublicSandboxMode } from '../lib/sandbox/mode.js'; import { hostSandboxRefusal, windowsSandboxMessage } from '../lib/sandbox/host-check.js'; import { prepareMcpBeforeFirstPoll } from '../lib/mcp-first-run.js'; +import { createSeatStop } from '../lib/seat-stop.js'; import { detectSeatbelt } from '../lib/sandbox/seatbelt.js'; import { describeUpstreamRefusal } from '../lib/upstream-refusal.js'; import { @@ -3612,12 +3613,16 @@ Docs: // SIGTERM is the daemon's idle-boundary handoff signal. `stop()` stops // future polls but lets the in-flight turn finish and ack before the // event loop drains; exiting here would cut a model turn in half. - const requestStop = () => { + // seat-stop also arms the supervised-group deadline before launchd's + // SIGKILL, and kills any group still live when the process exits. + const seatStop = createSeatStop({ name, stop }); + const requestStop = (signal) => { console.log(`\n${stamp()} [${name}] stopping after the current turn...`); - stop(); + seatStop.requestStop(signal); }; process.on('SIGINT', requestStop); process.on('SIGTERM', requestStop); + process.once('exit', seatStop.onExit); }); // ── detach (ADR-005) ────────────────────────────────────────────────────── diff --git a/cli/src/lib/adapters/claude.js b/cli/src/lib/adapters/claude.js index d7e0b7abad..02fab7bf77 100644 --- a/cli/src/lib/adapters/claude.js +++ b/cli/src/lib/adapters/claude.js @@ -1,5 +1,6 @@ import { claudeRuntimeInventory } from '../native-runtime-inventory.js'; import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group'; /** * claude adapter — wraps the local `claude` CLI as a Commonly agent. * @@ -291,12 +292,14 @@ const publicSkillPromptArgs = async (skillSources) => { return { args: text ? ['--append-system-prompt', text] : [], included }; }; -const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {} }) => new Promise((resolve, reject) => { - const proc = spawnImpl(cmd, args, { +// `ownGroup` is set only under the run supervisor, which kills the child's +// whole group; unsupervised, the spawn options are exactly what they were. +const runClaude = ({ cmd, args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false }) => new Promise((resolve, reject) => { + const proc = spawnInOwnGroup(spawnImpl, cmd, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], - }); + }, { ownGroup }); observeSpawn(proc, onSpawn, controls); let stdout = ''; let stderr = ''; @@ -906,6 +909,7 @@ export default { // the value a tail can echo back. credentials: spawnCredentials(spawnCtx), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }), // test seam only — do not use in production + ownGroup: wantsOwnProcessGroup(ctx), }); // TASK-440: the context this turn left, read from the transcript the // child wrote under its own env (an isolated HOME for public seats). @@ -957,6 +961,7 @@ export default { timeoutMs: ctx.timeoutMs || DEFAULT_TIMEOUT_MS, credentials: spawnCredentials(retryCtx), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(modelArgs, { modelFlag: '--model', effortFlag: '--effort' }), + ownGroup: wantsOwnProcessGroup(ctx), }); const contextTokens = claudeContextTokens({ cwd: ctx.cwd, sessionId: freshId, diff --git a/cli/src/lib/adapters/codex.js b/cli/src/lib/adapters/codex.js index f55cd44100..48588bca5f 100644 --- a/cli/src/lib/adapters/codex.js +++ b/cli/src/lib/adapters/codex.js @@ -1,5 +1,6 @@ import { codexRuntimeInventory } from '../native-runtime-inventory.js'; import { observeSpawn, controlsFromArguments } from '../runtime-observation.js'; +import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group'; /** * codex adapter — wraps the local `codex` CLI as a Commonly agent. * @@ -572,14 +573,15 @@ const CODEX_PROVIDER_KEY_ENVS = [ 'CODEX_API_KEY', ]; -const runCodex = ({ args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {} }) => new Promise((resolve, reject) => { // stdio: ['ignore', 'pipe', 'pipe'] — without this, child_process.spawn +// `ownGroup`: see runClaude in claude.js. +const runCodex = ({ args, cwd, env, timeoutMs, credentials = [], spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false }) => new Promise((resolve, reject) => { // stdio: ['ignore', 'pipe', 'pipe'] — without this, child_process.spawn // defaults stdin to a fresh pipe. Codex 0.125.0's `exec` then blocks on // `Reading additional input from stdin...` because it sees an open pipe // and waits for input that never arrives. Interactive runs are fine because // codex detects a TTY and uses the argv prompt directly. Setting stdin to // `'ignore'` gives codex /dev/null → immediate EOF → it falls back to the // argv prompt as intended. Surfaced live during ADR-005 Phase 2 smoke. - const proc = spawnImpl('codex', args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }); + const proc = spawnInOwnGroup(spawnImpl, 'codex', args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup }); observeSpawn(proc, onSpawn, controls); let stderr = ''; let timedOut = false; @@ -740,6 +742,7 @@ export default { // nothing left to compare a tail against. credentials: spawnCredentials(ctx, CODEX_PROVIDER_KEY_ENVS), spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: controlsFromArguments(args.slice(args.indexOf('exec') + (args[args.indexOf('exec') + 1] === 'resume' ? 3 : 1), -1), { modelFlag: '--model', valueFlags: ['-c', '-o', '--permission-profile'], codex: true }), // test seam only — do not use in production + ownGroup: wantsOwnProcessGroup(ctx), }); let text = ''; diff --git a/cli/src/lib/seat-stop.js b/cli/src/lib/seat-stop.js new file mode 100644 index 0000000000..b650dab298 --- /dev/null +++ b/cli/src/lib/seat-stop.js @@ -0,0 +1,96 @@ +/** + * How `agent run` stops on a signal while supervised process groups may be + * live (see @commonlyai/seats/process-group). + * + * SIGTERM keeps the in-flight turn: the run loop stops polling and lets the + * turn finish. A launchd seat is SIGKILLed ExitTimeOut seconds after SIGTERM, + * and that SIGKILL cannot be caught, so a supervised group (outside the seat's + * own group, which launchd kills) would outlive the seat. So on a launchd + * seat, SIGTERM also arms a deadline a margin before ExitTimeOut, and the exit + * path kills whatever groups are left when the turn ends first. + * + * The deadline is armed only when the seat's plist exists. A seat without one + * (the daemon spawns it) is stopped with SIGTERM and never SIGKILLed, so it + * relies on the exit path alone, and a reassignment or config change does not + * cut the in-flight turn. + * + * ExitTimeOut is read from the seat's plist and never written. In a plist that + * exists, an unset, zero or unreadable value is launchd's default of 20 s, + * because launchd will SIGKILL that seat. + */ +import { spawnSync } from 'child_process'; +import { existsSync, readFileSync } from 'fs'; +import { homedir } from 'os'; +import { join } from 'path'; +import { armSupervisedGroupDeadline, killSupervisedGroups } from '@commonlyai/seats/process-group'; + +export const DEFAULT_EXIT_TIMEOUT_S = 20; +const MAX_MARGIN_MS = 5000; +const SEAT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; + +/** The seat's launchd plist path, or null for a name that cannot be a file name. */ +export const seatPlistPath = (name, home = homedir()) => (typeof name === 'string' && SEAT_NAME.test(name) + ? join(home, 'Library', 'LaunchAgents', `me.commonly.seat-${name}.plist`) + : null); + +const positive = (value) => (Number.isFinite(value) && value > 0 ? value : null); + +/** + * ExitTimeOut in seconds, or null when the plist is missing, has none, or + * holds a value launchd would not use as a timeout. An XML plist is read + * directly; a binary one through `plutil -extract` (read-only). + */ +export const readExitTimeOut = (path, { readFile = readFileSync, run = spawnSync } = {}) => { + if (!path) return null; + let raw; + try { raw = readFile(path); } catch { return null; } + const text = Buffer.isBuffer(raw) ? raw.toString('utf8') : String(raw); + if (text.startsWith('bplist')) { + try { + const out = run('plutil', ['-extract', 'ExitTimeOut', 'raw', '-o', '-', path], { encoding: 'utf8', timeout: 5000 }); + return out.status === 0 ? positive(Number(String(out.stdout).trim())) : null; + } catch { return null; } + } + const match = /<key>\s*ExitTimeOut\s*<\/key>\s*<(integer|real)>\s*([0-9.]+)\s*<\/\1>/.exec(text); + return match ? positive(Number(match[2])) : null; +}; + +/** + * Whether the seat's plist exists, and its ExitTimeOut (null when unset or + * unreadable). Only an existing plist means launchd will SIGKILL this seat. + */ +export const readSeatPlist = (path, { exists = existsSync, ...deps } = {}) => { + if (!path || !exists(path)) return { present: false, exitTimeOut: null }; + return { present: true, exitTimeOut: readExitTimeOut(path, deps) }; +}; + +/** Milliseconds after SIGTERM to kill the groups: ExitTimeOut less a margin of 5 s or a quarter, whichever is less. */ +export const groupKillDeadlineMs = (exitTimeOutS) => { + const totalMs = (positive(exitTimeOutS) ?? DEFAULT_EXIT_TIMEOUT_S) * 1000; + return Math.floor(totalMs - Math.min(MAX_MARGIN_MS, totalMs / 4)); +}; + +/** + * `requestStop(signal)` stops the run loop and, on SIGTERM to a seat with a + * plist, arms the group deadline; `onExit` kills the groups left at exit. + * `deps` is a test seam. + */ +export const createSeatStop = ({ name, stop, home, deps = {} } = {}) => { + const { + arm = armSupervisedGroupDeadline, + kill = killSupervisedGroups, + readPlist = readSeatPlist, + } = deps; + return { + requestStop(signal) { + stop(); + if (signal !== 'SIGTERM') return null; + const plist = readPlist(seatPlistPath(name, home)); + if (!plist.present) return null; + const deadlineMs = groupKillDeadlineMs(plist.exitTimeOut); + arm(deadlineMs); + return deadlineMs; + }, + onExit() { kill(); }, + }; +}; diff --git a/packages/seats/src/index.js b/packages/seats/src/index.js index 6f0be90d3c..47e30b6def 100644 --- a/packages/seats/src/index.js +++ b/packages/seats/src/index.js @@ -1,6 +1,11 @@ export { createRegistry, providerKeyEnvNames } from './contract.js'; export { requireSeatModel } from './seat-model.js'; export { controlsFromArguments, observeSpawn } from './runtime-observation.js'; +export { + LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP, spawnInOwnGroup, wantsOwnProcessGroup, leadsOwnGroup, killProcessTree, + trackSupervisedGroup, killSupervisedGroups, armSupervisedGroupDeadline, + readProcessTable, readProcess, descendantsOf, killDescendants, +} from './process-group.js'; export { adapterFailure, spawnCredentials } from './upstream-refusal.js'; export { default as stub } from './adapters/stub.js'; export { createHermesAdapter } from './adapters/hermes.js'; diff --git a/packages/seats/src/process-group.js b/packages/seats/src/process-group.js new file mode 100644 index 0000000000..286b856e23 --- /dev/null +++ b/packages/seats/src/process-group.js @@ -0,0 +1,261 @@ +/** + * Process groups for supervised children. + * + * An agent CLI spawns its own children: MCP servers through `npx`, a native + * binary behind a node launcher, shells for tools. Killing only the CLI leaves + * those running, holding the run's work and keys on a shared host. A child the + * supervisor will kill is therefore started as the leader of its own process + * group, and killed as a group. + * + * Leading a group is never inferred. The spawner that passed `detached` marks + * the child, and only a marked child that is still alive is ever signalled by + * negative pid: signalling `-pid` for a process that does not lead its group + * can reach an unrelated group. + * + * A group kill misses a descendant that moved to a group of its own (codex + * starts every MCP server and tool command that way; claude's Bash tool does + * too). So before any signal, the kill also snapshots the process table and + * walks the child's descendants, deepest first. Each pid is re-read right + * before it is signalled and killed only if its parent and start time still + * match the snapshot: a pid that exited and was reused by another process on + * this host is left alone. Leaves go first so a pid's parent is still alive, + * and its ppid still meaningful, when it is checked. + * + * What the walk does not catch, by design, because each case would need the + * ppid match loosened: + * (a) a descendant reparented to 1 (its parent died) between the snapshot + * and its re-read is skipped: its ppid no longer matches; + * (b) a process started after the snapshot is not in it, and one that calls + * setsid after the snapshot leaves the group the group kill signals; + * either is missed. + * + * A child in its own group is outside the wrapper's group, so launchd's kill + * of the seat's group on exit no longer reaches it. `trackSupervisedGroup` + * covers that: while any tracked group is live, the wrapper kills them all + * when it exits, and on SIGTERM, SIGINT or SIGHUP when nothing else handles + * the signal. A host that handles the signal itself (`agent run` finishes the + * in-flight turn on SIGTERM) owns the timing: it calls + * `armSupervisedGroupDeadline` so the groups die before launchd's SIGKILL, + * which no handler can catch, and the exit hook covers the clean case. + */ + +import { spawnSync } from 'node:child_process'; + +/** Set on a child by the code that spawned it with `detached: true`. */ +export const LEADS_PROCESS_GROUP = Symbol.for('@commonlyai/seats.leadsProcessGroup'); + +/** Whether a child can lead its own group here (POSIX only). */ +export const PROCESS_GROUPS_SUPPORTED = process.platform !== 'win32'; + +/** + * The ctx key the supervisor sets to tell an adapter it will kill the child's + * group, so the adapter may start the child in one. Unset, an adapter spawns + * exactly as it would unsupervised. + */ +export const OWN_PROCESS_GROUP = 'ownProcessGroup'; + +export const wantsOwnProcessGroup = (ctx) => ctx?.[OWN_PROCESS_GROUP] === true && PROCESS_GROUPS_SUPPORTED; + +/** + * Spawn through `spawnImpl`. With `ownGroup`, the child is started detached + * (it calls setsid, so it leads a new group whose id is its pid) and marked. + * Without it, `options` reach `spawnImpl` untouched. + */ +export const spawnInOwnGroup = (spawnImpl, command, args, options, { ownGroup = false } = {}) => { + if (!ownGroup || !PROCESS_GROUPS_SUPPORTED) return spawnImpl(command, args, options); + const child = spawnImpl(command, args, { ...options, detached: true }); + if (child && typeof child === 'object') child[LEADS_PROCESS_GROUP] = true; + return child; +}; + +const isAlive = (child) => child.exitCode === null && child.signalCode === null; + +/** + * True only for a child its spawner marked, with a real pid, that has not been + * reaped. Node sets exitCode/signalCode in the same tick it reaps, so while + * this holds the pid (and so the group id) cannot have been reused. + */ +export const leadsOwnGroup = (child) => !!child + && child[LEADS_PROCESS_GROUP] === true + && Number.isInteger(child.pid) && child.pid > 0 + && isAlive(child); + +// ── The process table ────────────────────────────────────────────────────── + +const PS_ENV = { ...process.env, LC_ALL: 'C' }; + +const parseTable = (text) => { + const rows = []; + for (const line of String(text || '').split('\n')) { + const match = /^\s*(\d+)\s+(\d+)\s+(\S.*?)\s*$/.exec(line); + if (match) rows.push({ pid: Number(match[1]), ppid: Number(match[2]), started: match[3] }); + } + return rows; +}; + +const ps = (args) => { + try { + const out = spawnSync('ps', args, { encoding: 'utf8', env: PS_ENV, timeout: 5000 }); + return out.error ? null : parseTable(out.stdout); + } catch { return null; } +}; + +/** Every process: { pid, ppid, started }. null when ps is unavailable. */ +export const readProcessTable = () => ps(['-A', '-o', 'pid=,ppid=,lstart=']); + +/** One process as it is now, or null when it is gone. */ +export const readProcess = (pid) => (ps(['-o', 'pid=,ppid=,lstart=', '-p', String(pid)]) || []) + .find((row) => row.pid === pid) || null; + +/** The descendants of `root` in `table`, deepest first. The root is not included. */ +export const descendantsOf = (table, root) => { + const byParent = new Map(); + for (const row of table || []) { + if (row.pid === row.ppid) continue; + if (!byParent.has(row.ppid)) byParent.set(row.ppid, []); + byParent.get(row.ppid).push(row); + } + const out = []; + const seen = new Set([root]); + let level = [root]; + for (let depth = 1; level.length; depth += 1) { + const next = []; + for (const parent of level) { + for (const row of byParent.get(parent) || []) { + if (seen.has(row.pid)) continue; + seen.add(row.pid); + out.push({ ...row, depth }); + next.push(row.pid); + } + } + level = next; + } + return out.sort((a, b) => b.depth - a.depth); +}; + +const DEFAULT_IO = Object.freeze({ table: readProcessTable, read: readProcess }); + +/** + * Signal each snapshotted descendant, deepest first, re-reading it just + * before: it is signalled only while its ppid and start time still match. + * Returns the pids signalled. + */ +export const killDescendants = (descendants, signal = 'SIGKILL', { read = readProcess } = {}) => { + const killed = []; + for (const row of descendants) { + const now = read(row.pid); + if (!now || now.ppid !== row.ppid || now.started !== row.started) continue; + try { process.kill(row.pid, signal); killed.push(row.pid); } catch { /* gone */ } + } + return killed; +}; + +const liveRoot = (child) => !!child && Number.isInteger(child.pid) && child.pid > 0 && isAlive(child); + +const killOne = (child, signal, table, io) => { + // The root must be alive, so unreaped, so its pid still names this child. + if (table && liveRoot(child)) killDescendants(descendantsOf(table, child.pid), signal, io); + if (leadsOwnGroup(child)) { + try { process.kill(-child.pid, signal); return true; } catch { /* fall back to the leader */ } + } + try { return child.kill(signal) !== false; } catch { return false; } +}; + +/** + * Kill a child, its verified descendants, and its whole group when it leads + * one. The table is read before any signal. Falls back to `child.kill` when + * the child does not lead a group, or the group signal fails. Returns whether + * the child was signalled. `io` is a test seam ({ table, read }). + */ +export const killProcessTree = (child, signal = 'SIGKILL', io = {}) => { + const seam = { ...DEFAULT_IO, ...io }; + return killOne(child, signal, liveRoot(child) ? seam.table() : null, seam); +}; + +// ── Wrapper-exit cleanup ──────────────────────────────────────────────────── + +const EXIT_SIGNALS = ['SIGTERM', 'SIGINT', 'SIGHUP']; +const tracked = new Set(); +const handlers = new Map(); +let onExit = null; +let deadline = null; + +/** SIGKILL every tracked group still led by a live child, with its verified descendants. Synchronous. */ +export const killSupervisedGroups = (signal = 'SIGKILL', io = {}) => { + const live = [...tracked].filter(leadsOwnGroup); + if (live.length === 0) return 0; + const seam = { ...DEFAULT_IO, ...io }; + const table = seam.table(); + let killed = 0; + for (const child of live) if (killOne(child, signal, table, seam)) killed += 1; + return killed; +}; + +/** + * Kill the tracked groups in `ms`, for a host that keeps running after a + * signal (to finish a turn) but will be SIGKILLed at a known deadline. The + * earliest deadline armed wins. The timer does not hold the process open. + * Returns a function that disarms it. + */ +export const armSupervisedGroupDeadline = (ms) => { + const wait = Math.max(0, Number(ms) || 0); + const at = Date.now() + wait; + if (deadline && deadline.at <= at) return deadline.cancel; + if (deadline) deadline.cancel(); + const timer = setTimeout(() => { deadline = null; killSupervisedGroups(); }, wait); + timer.unref?.(); + const entry = { at, cancel: () => { clearTimeout(timer); if (deadline === entry) deadline = null; } }; + deadline = entry; + return entry.cancel; +}; + +const uninstall = () => { + for (const [signal, handler] of handlers) process.removeListener(signal, handler); + handlers.clear(); + if (onExit) process.removeListener('exit', onExit); + onExit = null; +}; + +const install = () => { + if (onExit) return; + onExit = () => { killSupervisedGroups(); }; + process.on('exit', onExit); + for (const signal of EXIT_SIGNALS) { + const handler = () => { + // A host that handles this signal has taken over shutdown: it may be + // finishing the turn these groups belong to, so they are left alone and + // the host's deadline or exit kills them. + if (process.listenerCount(signal) !== 1) return; + killSupervisedGroups(); + // Listening for a signal replaces its default action. With nothing else + // handling it, restore that action and re-raise, so the wrapper still + // terminates exactly as it would have without this module. + uninstall(); + process.kill(process.pid, signal); + }; + handlers.set(signal, handler); + process.on(signal, handler); + } +}; + +const untrack = (child) => { + tracked.delete(child); + if (tracked.size === 0) uninstall(); +}; + +/** + * Track a group-leading child until it exits. The handlers exist only while a + * tracked child is live, so a wrapper with no supervised group in flight keeps + * its signal behaviour unchanged. A child that does not lead a group is + * ignored: launchd's group kill already reaches it. + */ +export const trackSupervisedGroup = (child) => { + if (!leadsOwnGroup(child) || tracked.has(child)) return false; + tracked.add(child); + install(); + child.once?.('exit', () => untrack(child)); + return true; +}; + +/** For tests: how many groups are tracked. */ +export const trackedGroupCount = () => tracked.size; diff --git a/packages/seats/src/supervisor.js b/packages/seats/src/supervisor.js index 513a420432..3604a271ed 100644 --- a/packages/seats/src/supervisor.js +++ b/packages/seats/src/supervisor.js @@ -8,9 +8,16 @@ * the adapter emits, and any byte on the stdout/stderr of a child the adapter * reports through `ctx.onSpawn`. A run that is busy but quiet past the window * is indistinguishable from a hung one, which is why the window is long. + * + * The adapter is told (ctx.ownProcessGroup) that it may start its child as the + * leader of a new process group; a child it marks as one is killed as a group, + * so the CLI's own children (MCP servers, a native binary behind a launcher) + * die with it. See process-group.js, which also kills those groups when the + * wrapper itself is signalled or exits. */ import { RunFailure, runFailure, failureKindOf, FAILURE_KINDS } from './failures.js'; import { redactKnownCredentials, scrubAdapterFailure, spawnCredentials } from './upstream-refusal.js'; +import { killProcessTree, trackSupervisedGroup, OWN_PROCESS_GROUP } from './process-group.js'; export const SUPERVISOR_DEFAULTS = Object.freeze({ // Longer than the quiet stretches a healthy turn has: a long-context first @@ -110,7 +117,7 @@ const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, e let killed = 0; for (const child of children) { if (!isAlive(child)) continue; - try { child.kill('SIGKILL'); killed += 1; } catch { /* already gone */ } + if (killProcessTree(child, 'SIGKILL')) killed += 1; } return killed; }; @@ -180,7 +187,11 @@ const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, e }; // Wraps the caller's onSpawn: every argument reaches it unchanged. + // A group-leading child is tracked for the wrapper's exit even when it + // arrives after this attempt ended: it is outside the seat's own group, so + // nothing else would kill it. const onSpawn = (...args) => { + for (const child of args.filter(isChild)) trackSupervisedGroup(child); if (live) { touch(); for (const child of args.filter(isChild)) { @@ -206,7 +217,7 @@ const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, e let pending; try { pending = Promise.resolve(adapter.spawn(prompt, { - ...ctx, ...(sessionId ? { sessionId } : {}), trace, signal: controller.signal, onSpawn, + ...ctx, ...(sessionId ? { sessionId } : {}), trace, signal: controller.signal, onSpawn, [OWN_PROCESS_GROUP]: true, })); } catch (error) { pending = Promise.reject(error); @@ -225,7 +236,8 @@ const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, e * @param {object} args * @param {import('./contract.js').Adapter} args.adapter * @param {string} args.prompt - * @param {object} [args.ctx] passed to adapter.spawn; `trace`, `signal` and `onSpawn` are wrapped. + * @param {object} [args.ctx] passed to adapter.spawn; `trace`, `signal` and `onSpawn` are wrapped, + * and `ownProcessGroup` is set to true. * `ctx.signal`, when given, cancels the run (outcome `cancelled`). * @param {ReturnType<import('./trace/store.js').createTraceStore>} args.store * @param {object} [args.policy] diff --git a/packages/seats/test-support/processes.js b/packages/seats/test-support/processes.js new file mode 100644 index 0000000000..5e2811f8c4 --- /dev/null +++ b/packages/seats/test-support/processes.js @@ -0,0 +1,30 @@ +// Real-process helpers for the kill tests. +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; + +// A zombie still answers kill(pid, 0); on Linux it is read as gone, because a +// SIGKILLed orphan waits there until init reaps it. +export const isRunning = (pid) => { + try { process.kill(pid, 0); } catch (error) { return error.code !== 'ESRCH'; } + try { return !/^\d+ \(.*\) Z/.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return true; } +}; + +export const waitFor = async (predicate, ms = 5000) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (await predicate()) return true; + await new Promise((r) => { setTimeout(r, 25); }); + } + return predicate(); +}; + +/** The comma-separated pids a process wrote to `file`, once it has written them. */ +export const readPids = async (file, count = 1) => { + const ready = () => existsSync(file) && new RegExp(`^\\d+(,\\d+){${count - 1}}\\s*$`).test(readFileSync(file, 'utf8')); + assert.ok(await waitFor(ready), `no ${count} pid(s) in ${file}`); + return readFileSync(file, 'utf8').trim().split(',').map(Number); +}; + +export const readPid = async (file) => (await readPids(file, 1))[0]; + +export const reap = (pid) => { try { process.kill(pid, 'SIGKILL'); } catch { /* gone */ } }; diff --git a/packages/seats/test/supervisor-group-kill.test.js b/packages/seats/test/supervisor-group-kill.test.js new file mode 100644 index 0000000000..4de7ab9211 --- /dev/null +++ b/packages/seats/test/supervisor-group-kill.test.js @@ -0,0 +1,218 @@ +// A stalled turn's kill has to reach what the agent CLI spawned, not only the +// CLI. Each test here uses real processes: a shell that starts a grandchild +// (`sleep 300 &`), writes its pid, and waits on it, which is the shape of a CLI +// that started an MCP server. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { isRunning, readPid, reap, waitFor } from '../test-support/processes.js'; +import { + LEADS_PROCESS_GROUP, OWN_PROCESS_GROUP, leadsOwnGroup, spawnInOwnGroup, trackedGroupCount, wantsOwnProcessGroup, +} from '../src/process-group.js'; + +const SRC = fileURLToPath(new URL('../src/', import.meta.url)); +const POSIX = process.platform !== 'win32'; +const STDIO = ['ignore', 'pipe', 'pipe']; + + +/** + * An adapter shaped like the real ones: it spawns through spawnInOwnGroup, + * asking for a group only when the supervisor offered one, and reports the + * child through onSpawn. It ignores the abort, so the supervisor's kill is the + * only thing that ends the turn. + */ +const grandchildAdapter = ({ pidFile, children, honourGroup = true }) => ({ + name: 'sh-grandchild', + spawn(prompt, ctx) { + return new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', `sleep 300 & echo $! > '${pidFile}'; wait`], { stdio: STDIO }, { + ownGroup: honourGroup && wantsOwnProcessGroup(ctx), + }); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + child.on('close', (code, signal) => { + if (ctx.signal.aborted) reject(ctx.signal.reason); + else reject(runFailure('exited', `sh exited ${code ?? signal}`)); + }); + }); + }, +}); + +const withHome = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-group-kill-')); + try { return await fn(home); } finally { rmSync(home, { recursive: true, force: true }); } +}; + +const STALL = { stallMs: 300, cancelGraceMs: 300, timeoutMs: 20000 }; + +test('the supervisor tells the adapter it may lead its own group', async () => withHome(async (home) => { + let seen; + const adapter = { name: 'spy', spawn: async (prompt, ctx) => { seen = ctx; return { text: 'ok' }; } }; + await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: join(home, 'trace') }) }); + assert.equal(seen[OWN_PROCESS_GROUP], true); +})); + +test('unasked, spawnInOwnGroup hands the options through untouched', () => { + const options = { cwd: '/x', env: { A: '1' }, stdio: STDIO }; + const calls = []; + const child = spawnInOwnGroup((...args) => { calls.push(args); return {}; }, 'cmd', ['a'], options); + assert.equal(calls[0][2], options); + assert.equal(Object.hasOwn(calls[0][2], 'detached'), false); + assert.equal(child[LEADS_PROCESS_GROUP], undefined); +}); + +test('a group-leading child is never inferred: detached without the marker is not a leader', { skip: !POSIX }, async () => { + const child = spawn('sleep', ['300'], { detached: true, stdio: 'ignore' }); + try { + await once(child, 'spawn'); + assert.equal(leadsOwnGroup(child), false); + child[LEADS_PROCESS_GROUP] = true; + assert.equal(leadsOwnGroup(child), true); + } finally { + child.kill('SIGKILL'); + await once(child, 'exit'); + } + assert.equal(leadsOwnGroup(child), false, 'a reaped child leads nothing, marked or not'); +}); + +test('a stalled turn\'s kill reaches the grandchild of a group-leading child', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + let grandchild; + try { + const run = superviseRun({ + adapter: grandchildAdapter({ pidFile, children }), prompt: 'p', ctx: {}, + store: createTraceStore({ root: join(home, 'trace') }), policy: STALL, + }); + grandchild = await readPid(pidFile); + assert.equal(leadsOwnGroup(children[0]), true); + assert.equal(isRunning(grandchild), true); + const out = await run; + assert.equal(out.outcome, 'stuck'); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} survived the kill`); + assert.ok(await waitFor(() => children[0].exitCode !== null || children[0].signalCode !== null), 'the leader survived'); + assert.ok(await waitFor(() => trackedGroupCount() === 0), 'the exited group is still tracked'); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +test('a child that does not lead its group is killed with child.kill, and no negative pid is signalled', { skip: !POSIX }, async (t) => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + const groupSignals = []; + const realKill = process.kill; + t.mock.method(process, 'kill', function kill(pid, signal) { + if (Number(pid) < 0) groupSignals.push([pid, signal]); + return realKill.call(process, pid, signal); + }); + let grandchild; + try { + const out = await (async () => { + const run = superviseRun({ + adapter: grandchildAdapter({ pidFile, children, honourGroup: false }), prompt: 'p', ctx: {}, + store: createTraceStore({ root: join(home, 'trace') }), policy: STALL, + }); + grandchild = await readPid(pidFile); + return run; + })(); + assert.equal(out.outcome, 'stuck'); + assert.equal(leadsOwnGroup(children[0]), false); + assert.ok(await waitFor(() => children[0].signalCode !== null), 'child.kill did not reach the child'); + assert.equal(children[0].signalCode, 'SIGKILL'); + assert.deepEqual(groupSignals, []); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +// ── The wrapper's own exit ────────────────────────────────────────────────── +// A group-leading child is outside the wrapper's group, so launchd's kill of +// the seat's group no longer reaches it. The wrapper itself has to kill it. + +const WRAPPER = (pidFile, mode) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { superviseRun } from ${JSON.stringify(pathToFileURL(join(SRC, 'supervisor.js')).href)}; +import { createTraceStore } from ${JSON.stringify(pathToFileURL(join(SRC, 'trace/store.js')).href)}; +import { observeSpawn } from ${JSON.stringify(pathToFileURL(join(SRC, 'runtime-observation.js')).href)}; +import { spawnInOwnGroup, wantsOwnProcessGroup } from ${JSON.stringify(pathToFileURL(join(SRC, 'process-group.js')).href)}; +const mode = ${JSON.stringify(mode)}; +if (mode === 'host-handler') { + // Like 'agent run': the host handles the signal and decides when to exit. + process.on('SIGINT', () => { process.stdout.write('host-handled\\n'); setTimeout(() => process.exit(7), 300); }); +} +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise(() => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', "sleep 300 & echo $! > '${pidFile}'; wait"], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + if (mode === 'exit') process.exit(3); + }, 20); + }); +}) }; +superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(`${pidFile}.trace`)} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +`; + +const runWrapper = async (home, mode, signal) => { + const pidFile = join(home, `grandchild-${mode}-${signal || 'none'}.pid`); + const script = join(home, `wrapper-${mode}-${signal || 'none'}.mjs`); + writeFileSync(script, WRAPPER(pidFile, mode)); + const wrapper = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + let out = ''; + wrapper.stdout.on('data', (chunk) => { out += chunk; }); + const exited = once(wrapper, 'exit'); + let leader; + let grandchild; + try { + assert.ok(await waitFor(() => /ready \d+/.test(out)), `wrapper never got ready: ${out}`); + leader = Number(out.match(/ready (\d+)/)[1]); + grandchild = await readPid(pidFile); + if (signal) { + assert.equal(isRunning(grandchild), true); + wrapper.kill(signal); + } + const [code, how] = await Promise.race([exited, new Promise((r) => { setTimeout(() => r([null, 'timeout']), 5000); })]); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} survived the wrapper (${mode}, ${signal})`); + assert.ok(await waitFor(() => !isRunning(leader)), `leader ${leader} survived the wrapper (${mode}, ${signal})`); + return { code, how, out }; + } finally { + if (grandchild) reap(grandchild); + if (leader) reap(leader); + wrapper.kill('SIGKILL'); + } +}; + +for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) { + test(`a wrapper killed by ${signal} takes its supervised groups with it, and still dies by ${signal}`, { skip: !POSIX }, async () => withHome(async (home) => { + const { code, how } = await runWrapper(home, 'plain', signal); + assert.equal(code, null); + assert.equal(how, signal, 'the wrapper no longer dies by the signal it was sent'); + })); +} + +test('a wrapper that exits takes its supervised groups with it', { skip: !POSIX }, async () => withHome(async (home) => { + const { code } = await runWrapper(home, 'exit', null); + assert.equal(code, 3); +})); + +test('a wrapper that handles the signal itself keeps its groups through the signal, and loses them when it exits', { skip: !POSIX }, async () => withHome(async (home) => { + const { code, out } = await runWrapper(home, 'host-handler', 'SIGINT'); + assert.match(out, /host-handled/); + assert.equal(code, 7); +})); diff --git a/packages/seats/test/supervisor-tree-and-deadline.test.js b/packages/seats/test/supervisor-tree-and-deadline.test.js new file mode 100644 index 0000000000..eb2aee56a5 --- /dev/null +++ b/packages/seats/test/supervisor-tree-and-deadline.test.js @@ -0,0 +1,225 @@ +// Two layers on top of the group kill, with real processes. +// +// The descendant walk: a CLI's child that moved to a group of its own (codex +// does this for every MCP server and tool command) is outside the group kill, +// so the kill also walks the process tree. A pid is signalled only if its ppid +// and start time still match the snapshot, so a reused pid is left alone. +// +// The SIGTERM deadline: a host that keeps the in-flight turn on SIGTERM keeps +// the groups too, and kills them at a deadline set before launchd's SIGKILL. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { superviseRun } from '../src/supervisor.js'; +import { createTraceStore } from '../src/trace/store.js'; +import { runFailure } from '../src/failures.js'; +import { observeSpawn } from '../src/runtime-observation.js'; +import { + descendantsOf, killProcessTree, leadsOwnGroup, readProcess, readProcessTable, spawnInOwnGroup, wantsOwnProcessGroup, +} from '../src/process-group.js'; +import { isRunning, readPids, reap, waitFor } from '../test-support/processes.js'; + +const SRC = fileURLToPath(new URL('../src/', import.meta.url)); +const POSIX = process.platform !== 'win32'; + +const withHome = async (fn) => { + const home = mkdtempSync(join(tmpdir(), 'seats-tree-')); + try { return await fn(home); } finally { rmSync(home, { recursive: true, force: true }); } +}; + +// A child that starts `count` grandchildren, each the leader of a new group +// (detached), writes their pids, and stays alive. +const NEW_GROUP_PARENT = ` +const { spawn } = require('node:child_process'); +const pids = []; +for (let i = 0; i < Number(process.argv[2]); i += 1) { + pids.push(spawn('sleep', ['300'], { detached: true, stdio: 'ignore' }).pid); +} +require('node:fs').writeFileSync(process.argv[1], pids.join(',')); +setInterval(() => {}, 1e6); +`; + +const startParent = (pidFile, count, ownGroup) => spawnInOwnGroup( + spawn, process.execPath, ['-e', NEW_GROUP_PARENT, pidFile, String(count)], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup }, +); + +test('the descendant table is read deepest first', () => { + const table = [ + { pid: 10, ppid: 1, started: 'a' }, { pid: 11, ppid: 10, started: 'b' }, { pid: 12, ppid: 11, started: 'c' }, + { pid: 13, ppid: 10, started: 'd' }, { pid: 20, ppid: 1, started: 'e' }, + ]; + assert.deepEqual(descendantsOf(table, 10).map((row) => [row.pid, row.depth]), [[12, 2], [11, 1], [13, 1]]); +}); + +test('a stall kill reaches a grandchild that started a group of its own', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchild.pid'); + const children = []; + let grandchild; + try { + const adapter = { + name: 'new-group-parent', + spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = startParent(pidFile, 1, wantsOwnProcessGroup(ctx)); + children.push(child); + observeSpawn(child, ctx.onSpawn, {}); + child.on('close', () => reject(ctx.signal.aborted ? ctx.signal.reason : runFailure('exited', 'parent exited'))); + }), + }; + const run = superviseRun({ + adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: join(home, 'trace') }), + policy: { stallMs: 300, cancelGraceMs: 300, timeoutMs: 20000 }, + }); + [grandchild] = await readPids(pidFile, 1); + assert.equal(leadsOwnGroup(children[0]), true); + assert.equal(isRunning(grandchild), true); + assert.equal((await run).outcome, 'stuck'); + assert.ok(await waitFor(() => !isRunning(grandchild)), `grandchild ${grandchild} in its own group survived the kill`); + } finally { + if (grandchild) reap(grandchild); + for (const child of children) child.kill('SIGKILL'); + } +})); + +test('a pid reused by another process is left alone; the descendant the snapshot named is killed', { skip: !POSIX }, async () => withHome(async (home) => { + const pidFile = join(home, 'grandchildren.pid'); + const parent = startParent(pidFile, 2, true); + // An unrelated process: not in the parent's tree at all. + const unrelated = spawn('sleep', ['300'], { stdio: 'ignore' }); + const unrelatedStarted = once(unrelated, 'spawn'); + let real; + let reused; + try { + [real, reused] = await readPids(pidFile, 2); + await unrelatedStarted; + const table = readProcessTable(); + assert.ok(table, 'ps is unavailable, so the walk cannot run'); + assert.ok(readProcess(reused), 'the reused stand-in is not running'); + // The snapshot as it would read if each pid had belonged to an earlier + // process: `reused` is the right parent's child but started at another + // time, and `unrelated` is named as the parent's child though it is not. + const stub = table.map((row) => (row.pid === reused ? { ...row, started: 'Thu Jan 1 00:00:00 1970' } : row)); + stub.push({ pid: unrelated.pid, ppid: parent.pid, started: readProcess(unrelated.pid).started }); + + killProcessTree(parent, 'SIGKILL', { table: () => stub }); + + assert.ok(await waitFor(() => !isRunning(real)), 'the walk did not kill the descendant the snapshot named'); + assert.ok(await waitFor(() => parent.signalCode !== null), 'the parent survived'); + await new Promise((r) => { setTimeout(r, 200); }); + assert.equal(isRunning(reused), true, 'a pid whose start time changed was signalled'); + assert.equal(isRunning(unrelated.pid), true, 'a pid whose parent changed was signalled'); + } finally { + for (const pid of [real, reused]) if (pid) reap(pid); + unrelated.kill('SIGKILL'); + parent.kill('SIGKILL'); + } +})); + +// ── SIGTERM in a host that keeps the turn ───────────────────────────────────── + +// Shaped like `agent run`: SIGTERM asks for a stop after the current turn and +// arms the deadline; when the turn is over, the host's exit path kills the +// groups. The turn ends when the test creates `go`; its child then says +// `done` and lingers, as a CLI shutting down can. +const HOST = ({ pidFile, goFile, deadlineMs }) => ` +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { superviseRun } from ${JSON.stringify(pathToFileURL(join(SRC, 'supervisor.js')).href)}; +import { createTraceStore } from ${JSON.stringify(pathToFileURL(join(SRC, 'trace/store.js')).href)}; +import { observeSpawn } from ${JSON.stringify(pathToFileURL(join(SRC, 'runtime-observation.js')).href)}; +import { + armSupervisedGroupDeadline, killSupervisedGroups, spawnInOwnGroup, wantsOwnProcessGroup, +} from ${JSON.stringify(pathToFileURL(join(SRC, 'process-group.js')).href)}; +let stopping = false; +process.on('SIGTERM', () => { + stopping = true; + process.stdout.write('stopping\\n'); + armSupervisedGroupDeadline(${deadlineMs}); +}); +const script = "sleep 300 & echo $! > '${pidFile}'; while [ ! -f '${goFile}' ]; do sleep 0.05; done; echo done; wait"; +const adapter = { name: 'sh', spawn: (prompt, ctx) => new Promise((resolve, reject) => { + const child = spawnInOwnGroup(spawn, 'sh', ['-c', script], { stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup: wantsOwnProcessGroup(ctx) }); + observeSpawn(child, ctx.onSpawn, {}); + child.stdout.on('data', (chunk) => { if (String(chunk).includes('done')) resolve({ text: 'done' }); }); + child.on('close', (code, signal) => reject(new Error('sh ended ' + (code ?? signal)))); + child.once('spawn', () => { + const poll = setInterval(() => { + if (!existsSync('${pidFile}') || !/^\\d+\\s*$/.test(readFileSync('${pidFile}', 'utf8'))) return; + clearInterval(poll); + process.stdout.write('ready ' + child.pid + '\\n'); + }, 20); + }); +}) }; +const out = await superviseRun({ adapter, prompt: 'p', ctx: {}, store: createTraceStore({ root: ${JSON.stringify(`${pidFile}.trace`)} }), + policy: { stallMs: 0, timeoutMs: 0 } }); +process.stdout.write('turn ' + out.outcome + '\\n'); +if (stopping) { killSupervisedGroups(); process.exit(0); } +`; + +const startHost = async (home, deadlineMs) => { + const pidFile = join(home, 'grandchild.pid'); + const goFile = join(home, 'go'); + const script = join(home, 'host.mjs'); + writeFileSync(script, HOST({ pidFile, goFile, deadlineMs })); + const host = spawn(process.execPath, [script], { stdio: ['ignore', 'pipe', 'inherit'] }); + const state = { host, goFile, out: '' }; + host.stdout.on('data', (chunk) => { state.out += chunk; }); + state.exited = once(host, 'exit'); + assert.ok(await waitFor(() => /ready \d+/.test(state.out)), `host never got ready: ${state.out}`); + state.leader = Number(state.out.match(/ready (\d+)/)[1]); + [state.grandchild] = await readPids(pidFile, 1); + return state; +}; + +const cleanup = (state) => { + if (!state) return; + reap(state.grandchild); + reap(state.leader); + state.host.kill('SIGKILL'); +}; + +test('SIGTERM mid-turn: the turn completes, then the groups die', { skip: !POSIX }, async () => withHome(async (home) => { + let state; + try { + state = await startHost(home, 10000); + state.host.kill('SIGTERM'); + assert.ok(await waitFor(() => /stopping/.test(state.out)), 'the host never saw SIGTERM'); + await new Promise((r) => { setTimeout(r, 400); }); + assert.equal(isRunning(state.grandchild), true, 'SIGTERM killed the turn\'s group'); + assert.equal(isRunning(state.leader), true, 'SIGTERM killed the turn\'s child'); + writeFileSync(state.goFile, ''); + const [code] = await Promise.race([state.exited, new Promise((r) => { setTimeout(() => r(['timeout']), 5000); })]); + assert.match(state.out, /turn ok/); + assert.equal(code, 0); + assert.ok(await waitFor(() => !isRunning(state.grandchild)), 'the group outlived the host'); + assert.ok(await waitFor(() => !isRunning(state.leader)), 'the leader outlived the host'); + } finally { + cleanup(state); + } +})); + +test('SIGTERM with the turn still running: the deadline kills the groups before ExitTimeOut', { skip: !POSIX }, async () => withHome(async (home) => { + // Stands in for launchd: ExitTimeOut 2 s, deadline 1 s (the CLI derives the + // real one from the seat's plist). + const exitTimeOutMs = 2000; + const deadlineMs = 1000; + let state; + try { + state = await startHost(home, deadlineMs); + const sent = Date.now(); + state.host.kill('SIGTERM'); + assert.ok(await waitFor(() => /stopping/.test(state.out)), 'the host never saw SIGTERM'); + await new Promise((r) => { setTimeout(r, deadlineMs / 2); }); + assert.equal(isRunning(state.grandchild), true, 'the group died before its deadline'); + assert.ok(await waitFor(() => !isRunning(state.grandchild), exitTimeOutMs), 'the group outlived ExitTimeOut'); + const diedAfter = Date.now() - sent; + assert.ok(diedAfter < exitTimeOutMs, `the group died ${diedAfter}ms after SIGTERM, past ExitTimeOut`); + assert.ok(await waitFor(() => !isRunning(state.leader), 1000), 'the leader outlived its group'); + } finally { + cleanup(state); + } +})); From 20e459e992983f08677c4956459bbb1b155b4db5 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:52:28 -0400 Subject: [PATCH 20/20] fix(seats): superviseRun passes the run's actor, task and pod to the trace store (#2736) The supervisor started every run with only runId, adapter and label, so its own run.started entry, and any entry an adapter or the valet emitted without naming an actor, recorded actor: null, and no entry carried the board task or pod. superviseRun now takes taskId, podId and actor and hands them to store.startRun, which already validates them (a taskId without a podId is refused before anything spawns). Claude-Session: https://claude.ai/code/session_013ENddpGzhE7S8qjDSYAMuu Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> --- packages/seats/src/supervisor.js | 13 +++++++-- packages/seats/test/supervisor.test.js | 37 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/packages/seats/src/supervisor.js b/packages/seats/src/supervisor.js index 3604a271ed..7bfd471b1a 100644 --- a/packages/seats/src/supervisor.js +++ b/packages/seats/src/supervisor.js @@ -248,13 +248,22 @@ const runAttempt = ({ adapter, prompt, ctx, run, settings, attempt, sessionId, e * @param {(info: {runId, adapter, attempt, idleMs, windowMs}) => void} [args.policy.onStalled] * @param {string} [args.runId] * @param {string} [args.label] + * @param {string} [args.taskId] the board task this run works; needs `podId` + * @param {string} [args.podId] + * @param {object} [args.actor] the run's own party, {type, id, owner?, onBehalfOf?}; + * entries the run writes without naming an actor (run.started, tool calls, + * permission requests) record this one instead of null * @returns {Promise<{ runId: string, outcome: string, attempts: number, result?: object, failure?: Error }>} */ -export const superviseRun = async ({ adapter, prompt, ctx = {}, store, policy = {}, runId, label } = {}) => { +export const superviseRun = async ({ + adapter, prompt, ctx = {}, store, policy = {}, runId, label, taskId, podId, actor, +} = {}) => { if (!adapter || typeof adapter.spawn !== 'function') throw new Error('superviseRun needs an adapter with spawn()'); if (!store || typeof store.startRun !== 'function') throw new Error('superviseRun needs a trace store'); const settings = resolvePolicy(policy); - const run = store.startRun({ runId, adapter: adapter.name, label }); + const run = store.startRun({ + runId, adapter: adapter.name, label, taskId, podId, actor, + }); // The runtime token rides in ctx, not through the valet, so the store would // not otherwise know to scrub it from agent text. if (typeof run.redact === 'function' && typeof ctx.runtimeToken === 'string' && ctx.runtimeToken) { diff --git a/packages/seats/test/supervisor.test.js b/packages/seats/test/supervisor.test.js index 0c0e806d9b..7005ee8454 100644 --- a/packages/seats/test/supervisor.test.js +++ b/packages/seats/test/supervisor.test.js @@ -78,6 +78,43 @@ test('a successful run: started, the adapter\'s events, settled ok — in that o assert.equal(store.listRuns()[0].outcome, 'ok'); })); +test('the run\'s actor, task and pod reach the record, so run-owned entries are attributed', () => withStore(async (store) => { + const actor = { type: 'agent', id: 'u-agent-quire', owner: 'u-sam', onBehalfOf: { id: 'u-cy', role: 'assigner' } }; + const adapter = { + name: 'ok', + async spawn(prompt, ctx) { + ctx.trace.emit('agent.message_chunk', { text: 'PONG' }); + return { text: 'PONG' }; + }, + }; + const out = await superviseRun({ + adapter, prompt: 'ping', store, policy: FAST, taskId: 'TASK-548', podId: 'pod-sharpen', actor, + }); + assert.equal(out.outcome, 'ok'); + const entries = store.readRun(out.runId); + for (const entry of entries) { + assert.equal(entry.taskId, 'TASK-548', `${entry.type} carries the task`); + assert.equal(entry.podId, 'pod-sharpen', `${entry.type} carries the pod`); + } + const byType = Object.fromEntries(entries.map((e) => [e.type, e])); + assert.deepEqual(byType['run.started'].actor, actor, 'the supervisor\'s own run start is the run\'s'); + assert.deepEqual(byType['agent.message_chunk'].actor, actor, 'an emit that names no actor takes the run\'s'); + const [indexed] = store.listRuns(); + assert.equal(indexed.taskId, 'TASK-548'); + assert.equal(indexed.podId, 'pod-sharpen'); + assert.deepEqual(indexed.actor, actor); +})); + +test('a taskId without a podId is refused before the adapter is spawned', () => withStore(async (store) => { + let spawned = false; + const adapter = { name: 'ok', async spawn() { spawned = true; return { text: '' }; } }; + await assert.rejects( + superviseRun({ adapter, prompt: 'ping', store, policy: FAST, taskId: 'TASK-548' }), + /a taskId needs a podId/, + ); + assert.equal(spawned, false); +})); + test('a hung adapter that honours abort settles stuck, with the reason on its signal', () => withStore(async (store) => { const calls = []; const stalled = [];