diff --git a/backend/.eslintignore b/backend/.eslintignore index 5446bcda1c..63dd5764b2 100644 --- a/backend/.eslintignore +++ b/backend/.eslintignore @@ -9,4 +9,6 @@ test-discord-*.js scripts/deploy-discord-commands.js scripts/register-discord-commands.js services/discordService.js -services/discordCommandService.js \ No newline at end of file +services/discordCommandService.js +# Byte-identical copies of packages/seats (see vendor/seats/README.md). +vendor/ diff --git a/backend/__tests__/service/pgMigrations.test.js b/backend/__tests__/service/pgMigrations.test.js index 42907dae1b..e1395ed36c 100644 --- a/backend/__tests__/service/pgMigrations.test.js +++ b/backend/__tests__/service/pgMigrations.test.js @@ -7,6 +7,9 @@ const { createRealPgSchema } = require('../utils/realPgSchema'); const { waitFor } = require('../utils/testUtils'); const real = process.env.INTEGRATION_TEST === 'true' ? describe : describe.skip; +// Every migration that ships, in order. A new migration adds its name here, so +// two lanes claiming the same number conflict on this line. +const SHIPPED = ['0001_legacy_schema.sql', '0002_record_chains.sql']; real('Postgres migrations: real DDL, rollback and competing boots', () => { let database; let pool; @@ -23,7 +26,7 @@ real('Postgres migrations: real DDL, rollback and competing boots', () => { }); it('boots the shipped baseline on a fresh database and does not reapply it', async () => { - expect(await runPgMigrations(pool)).toEqual(['0001_legacy_schema.sql']); + expect(await runPgMigrations(pool)).toEqual(SHIPPED); const history = (await pool.query('SELECT * FROM schema_migrations')).rows; expect(history[0].applied_at).toBeInstanceOf(Date); expect(await runPgMigrations(pool)).toEqual([]); @@ -46,7 +49,7 @@ real('Postgres migrations: real DDL, rollback and competing boots', () => { INSERT INTO messages (pod_id, user_id, content) VALUES ('pod', 'user', 'old message'); INSERT INTO migration_records (name, details) VALUES ('thread-cutoff', '{"cutoff":"kept"}'); `); - expect(await runPgMigrations(pool)).toEqual(['0001_legacy_schema.sql']); + expect(await runPgMigrations(pool)).toEqual(SHIPPED); expect((await pool.query('SELECT id, content, thread_root_id, freshness_revision FROM messages')).rows) .toEqual([{ id: 1, content: 'old message', thread_root_id: null, freshness_revision: '0', diff --git a/backend/__tests__/service/podRecords.realPg.test.js b/backend/__tests__/service/podRecords.realPg.test.js new file mode 100644 index 0000000000..d1fdc5d3df --- /dev/null +++ b/backend/__tests__/service/podRecords.realPg.test.js @@ -0,0 +1,232 @@ +/* eslint-disable import/no-unresolved, import/extensions -- TypeScript modules expose CJS exports. */ +/** + * TASK-513: board writes reach a task's record, and the record reaches only + * the people it may. Real MongoDB for the board and the pod, real PostgreSQL + * for the chain. + */ +process.env.JWT_SECRET = process.env.JWT_SECRET || 'pod-records-test-secret'; + +jest.mock('../../services/taskEventService', () => { + const actual = jest.requireActual('../../services/taskEventService'); + return { ...actual, notifyPodAgents: jest.fn().mockResolvedValue(undefined) }; +}); +jest.mock('../../services/systemActionLineService', () => ({ + emitSystemActionLine: jest.fn().mockResolvedValue(undefined), +})); + +const express = require('express'); +const request = require('supertest'); +const Pod = require('../../models/Pod'); +const Task = require('../../models/Task'); +const User = require('../../models/User'); +const { AgentInstallation } = require('../../models/AgentRegistry'); +const { hash } = require('../../utils/secret'); +const { pool } = require('../../config/db-pg'); +const repository = require('../../repositories/recordChainRepository'); +const { flushRecordWrites } = require('../../services/recordService'); +const KernelWorkSweepService = require('../../services/kernelWorkSweepService').default; +const { verifyRecord } = require('../../vendor/seats/src/record/format'); +const { + setupMongoDb, closeMongoDb, clearMongoDb, setupPgDb, closePgDb, generateTestToken, +} = require('../utils/testUtils'); + +const real = process.env.INTEGRATION_TEST === 'true' ? describe : describe.skip; +const AGENT_TOKEN = 'cm_agent_pod_record_route'; + +real('task record route on real PostgreSQL and MongoDB', () => { + let app; + let owner; + let outsider; + let admin; + let agent; + let pod; + let otherPod; + const podIds = () => [String(pod._id), String(otherPod._id)]; + + beforeAll(async () => { + await setupMongoDb(); + await setupPgDb(); + await Task.init(); + app = express(); + app.use(express.json()); + app.use('/api/v1/tasks', require('../../routes/tasksApi')); + app.use('/api/pods', require('../../routes/podRecords')); + }); + + beforeEach(async () => { + await clearMongoDb(); + const stamp = Date.now(); + const person = (name, over = {}) => User.create({ + username: `${name}-${stamp}`, email: `${name}-${stamp}@test.com`, password: 'Password123!', verified: true, ...over, + }); + owner = await person('record-owner'); + outsider = await person('record-outsider'); + admin = await person('record-admin', { role: 'admin' }); + agent = await person('record-agent', { + username: 'record-agent', + email: `record-agent-${stamp}@agents.commonly.local`, + isBot: true, + botType: 'agent', + botMetadata: { agentName: 'record-agent', instanceId: 'default', ownerUserId: String(owner._id) }, + agentRuntimeTokens: [{ tokenHash: hash(AGENT_TOKEN), label: 'record', createdAt: new Date() }], + }); + pod = await Pod.create({ name: 'Record pod', type: 'team', createdBy: owner._id, members: [owner._id, agent._id] }); + otherPod = await Pod.create({ name: 'Other pod', type: 'team', createdBy: owner._id, members: [owner._id, agent._id] }); + await AgentInstallation.create({ + agentName: 'record-agent', instanceId: 'default', podId: pod._id, version: '1.0.0', installedBy: owner._id, + }); + await Task.create({ podId: pod._id, taskNum: 1, taskId: 'TASK-1', title: 'recorded task', status: 'pending' }); + process.env.RECORD_V1_PODS = podIds().join(','); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await flushRecordWrites(); + await pool.query( + 'DELETE FROM record_lines WHERE chain_id IN (SELECT id FROM record_chains WHERE pod_id = ANY($1))', + [podIds()], + ); + await pool.query('DELETE FROM record_chains WHERE pod_id = ANY($1)', [podIds()]); + delete process.env.RECORD_V1_PODS; + }); + + afterAll(async () => { + await clearMongoDb(); + await closeMongoDb(); + await closePgDb(); + }); + + const as = (token) => ({ Authorization: `Bearer ${token}` }); + const read = (token, podId = pod._id, taskId = 'TASK-1', query = '') => request(app) + .get(`/api/pods/${podId}/record/tasks/${taskId}${query}`).set(as(token)); + const claim = () => request(app).post(`/api/v1/tasks/${pod._id}/TASK-1/claim`).set(as(AGENT_TOKEN)).send({ continueAnyway: true }); + + it('mirrors the board: claim, renewal (no line), block, resume, complete — and the export verifies', async () => { + expect((await claim()).status).toBe(200); + expect((await claim()).status).toBe(200); + expect((await request(app).patch(`/api/v1/tasks/${pod._id}/TASK-1`).set(as(AGENT_TOKEN)).send({ status: 'blocked' })).status).toBe(200); + expect((await claim()).status).toBe(200); + expect((await request(app).post(`/api/v1/tasks/${pod._id}/TASK-1/complete`).set(as(generateTestToken(owner._id))).send({})).status).toBe(200); + await flushRecordWrites(); + + const res = await read(generateTestToken(owner._id)); + expect(res.status).toBe(200); + expect(res.headers['content-type']).toMatch(/^application\/x-ndjson/); + expect(res.headers['cache-control']).toBe('no-store'); + const verified = verifyRecord(res.text); + expect(verified).toMatchObject({ ok: true, entries: 4, settled: false }); + expect(res.headers['x-record-head']).toBe(verified.head); + expect(res.headers['x-record-entries']).toBe('4'); + const lines = res.text.trimEnd().split('\n').map((line) => JSON.parse(line)); + expect(lines.map((l) => [l.kind, l.data])).toEqual([ + ['task.claimed', { leaseUntil: expect.any(String) }], + ['task.released', { reason: 'blocked' }], + ['task.claimed', { leaseUntil: expect.any(String) }], + ['task.done', { doneBy: String(owner._id) }], + ]); + const agentActor = { type: 'agent', id: String(agent._id), owner: String(owner._id) }; + expect(lines[0].actor).toEqual(agentActor); + expect(lines[1].actor).toEqual(agentActor); + expect(lines[3].actor).toEqual({ type: 'system', id: 'board' }); + expect(new Set(lines.map((l) => `${l.podId}/${l.taskId}/${l.runId}`)).size).toBe(1); + expect(lines[0].podId).toBe(String(pod._id)); + + // An agent member reads the same bytes; OTLP puts the task in one trace. + expect((await read(AGENT_TOKEN)).text).toBe(res.text); + const otlp = await read(AGENT_TOKEN, pod._id, 'TASK-1', '?format=otlp'); + expect(otlp.status).toBe(200); + expect(otlp.body.resourceSpans[0].scopeSpans[0].spans[0]).toMatchObject({ name: 'seats.run' }); + }); + + it('records the lease lapse, the claim a note restores, and the completion, in that order', async () => { + expect((await claim()).status).toBe(200); + const lapsedAt = new Date(Date.now() + 2 * 60 * 60 * 1000); + const { rescued } = await KernelWorkSweepService.rescueLapsed(lapsedAt); + expect(rescued).toHaveLength(1); + const note = await request(app).post(`/api/v1/tasks/${pod._id}/TASK-1/updates`).set(as(AGENT_TOKEN)).send({ text: 'still on it' }); + expect(note).toMatchObject({ status: 200, body: { leaseRenewed: true } }); + expect((await request(app).post(`/api/v1/tasks/${pod._id}/TASK-1/complete`).set(as(generateTestToken(owner._id))).send({})).status).toBe(200); + await flushRecordWrites(); + + const res = await read(generateTestToken(owner._id)); + expect(verifyRecord(res.text)).toMatchObject({ ok: true, entries: 4 }); + const lines = res.text.trimEnd().split('\n').map((line) => JSON.parse(line)); + expect(lines.map((l) => [l.kind, l.actor && l.actor.type, l.data.reason])).toEqual([ + ['task.claimed', 'agent', undefined], + ['task.released', 'system', 'lease_expired'], + ['task.claimed', 'agent', undefined], + ['task.done', 'system', undefined], + ]); + }); + + it('a PATCH to claimed records the claim without reporting the row\'s older lease as its own', async () => { + expect((await claim()).status).toBe(200); + const asOwner = as(generateTestToken(owner._id)); + expect((await request(app).patch(`/api/v1/tasks/${pod._id}/TASK-1`).set(asOwner).send({ status: 'blocked' })).status).toBe(200); + // The first claim's lease is still on the row; this PATCH sets none. + expect((await Task.findOne({ podId: pod._id, taskId: 'TASK-1' }).lean()).claimExpiresAt).toBeInstanceOf(Date); + expect((await request(app).patch(`/api/v1/tasks/${pod._id}/TASK-1`).set(asOwner).send({ status: 'in_progress' })).status).toBe(200); + await flushRecordWrites(); + const res = await read(generateTestToken(owner._id)); + const lines = res.text.trimEnd().split('\n').map((line) => JSON.parse(line)); + expect(lines.map((l) => [l.kind, l.data])).toEqual([ + ['task.claimed', { leaseUntil: expect.any(String) }], + ['task.released', { reason: 'blocked' }], + ['task.claimed', {}], + ]); + expect(lines[2].actor).toEqual({ type: 'human', id: String(owner._id) }); + }); + + it('refuses everyone else before any record byte is read', async () => { + await claim(); + await flushRecordWrites(); + const readChain = jest.spyOn(repository, 'readChain'); + + // Not a viewer of the pod: indistinguishable from no such record. + expect(await read(generateTestToken(outsider._id))).toMatchObject({ status: 404, body: { error: 'record_not_found' } }); + // An admin can view the pod, but the lines carry approver ids verbatim. + expect(await read(generateTestToken(admin._id))).toMatchObject({ status: 403, body: { error: 'record_requires_membership' } }); + // Task ids restart in each pod: TASK-1 belongs to the first pod only. + expect(await read(generateTestToken(owner._id), otherPod._id)).toMatchObject({ status: 404 }); + expect(await read(generateTestToken(owner._id), pod._id, 'TASK-404')).toMatchObject({ status: 404 }); + expect(await read(generateTestToken(owner._id), 'not-a-pod')).toMatchObject({ status: 404 }); + expect(await read(generateTestToken(owner._id), pod._id, 'bad%20id')).toMatchObject({ status: 404 }); + expect((await request(app).get(`/api/pods/${pod._id}/record/tasks/TASK-1`)).status).toBe(401); + // A pod the record is not enabled for serves nothing. + process.env.RECORD_V1_PODS = String(otherPod._id); + expect(await read(generateTestToken(owner._id))).toMatchObject({ status: 404 }); + expect(readChain).not.toHaveBeenCalled(); + + // Removing a member removes their access on the next read. + process.env.RECORD_V1_PODS = podIds().join(','); + await Pod.updateOne({ _id: pod._id }, { $pull: { members: agent._id } }); + expect(await read(AGENT_TOKEN)).toMatchObject({ status: 404 }); + expect(readChain).not.toHaveBeenCalled(); + }); + + it('answers a store outage with 503, never an empty record', async () => { + jest.spyOn(repository, 'readChain').mockRejectedValueOnce(new repository.RecordStoreUnavailableError()); + expect(await read(generateTestToken(owner._id))).toMatchObject({ status: 503, body: { error: 'record_store_unavailable' } }); + jest.spyOn(repository, 'readChain').mockRejectedValueOnce(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })); + jest.spyOn(console, 'error').mockImplementation(() => {}); + expect(await read(generateTestToken(owner._id))).toMatchObject({ status: 503, body: { error: 'record_store_unavailable' } }); + }); + + it('a failed record write leaves the board write as it was', async () => { + jest.spyOn(repository, 'appendToChain').mockRejectedValueOnce(new Error('pg down')); + jest.spyOn(console, 'warn').mockImplementation(() => {}); + const res = await claim(); + await flushRecordWrites(); + expect(res.status).toBe(200); + expect((await Task.findOne({ podId: pod._id, taskId: 'TASK-1' }).lean()).status).toBe('claimed'); + expect((await read(generateTestToken(owner._id))).headers['x-record-entries']).toBe('0'); + }); + + it('writes nothing for a pod the record is not enabled for', async () => { + process.env.RECORD_V1_PODS = String(otherPod._id); + expect((await claim()).status).toBe(200); + await flushRecordWrites(); + const { rows } = await pool.query('SELECT count(*)::int AS n FROM record_chains WHERE pod_id = $1', [String(pod._id)]); + expect(rows[0].n).toBe(0); + }); +}); diff --git a/backend/__tests__/service/recordChains.realPg.test.js b/backend/__tests__/service/recordChains.realPg.test.js new file mode 100644 index 0000000000..1a8b72e3f0 --- /dev/null +++ b/backend/__tests__/service/recordChains.realPg.test.js @@ -0,0 +1,168 @@ +/* eslint-disable import/no-unresolved, import/extensions -- TypeScript modules expose CJS exports. */ +/** + * TASK-513: the server's record chains on real PostgreSQL. Row locks, + * unique constraints and transaction rollback are what keep a chain whole + * across replicas, and pg-mem models none of them faithfully. + */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { runPgMigrations } = require('../../config/run-pg-migrations'); +const { createRealPgSchema } = require('../utils/realPgSchema'); +const { appendRecord, readRecordExport, BOARD_ACTOR } = require('../../services/recordService'); +const { appendToChain } = require('../../repositories/recordChainRepository'); +const { verifyRecord } = require('../../vendor/seats/src/record/format'); + +const real = process.env.INTEGRATION_TEST === 'true' ? describe : describe.skip; +const MIGRATIONS = path.join(__dirname, '../../config/migrations'); +const POD = '6a692a1be833c668acdb84cf'; +const AGENT = { type: 'agent', id: '6a920d43b08dee6c42a81c9e', owner: '6a47891ee025ce2e0ca1d30b' }; + +real('record chains on real PostgreSQL', () => { + let database; + let pool; + beforeEach(async () => { + database = await createRealPgSchema(); + pool = database.pool; + }); + afterEach(async () => { + if (database) await database.close(); + }); + + const kinds = (jsonl) => jsonl.trimEnd().split('\n').map((line) => JSON.parse(line)); + + it('upgrades a populated 0001 schema without touching its rows', async () => { + const legacy = fs.mkdtempSync(path.join(os.tmpdir(), 'record-0001-')); + try { + fs.copyFileSync(path.join(MIGRATIONS, '0001_legacy_schema.sql'), path.join(legacy, '0001_legacy_schema.sql')); + expect(await runPgMigrations(pool, legacy)).toEqual(['0001_legacy_schema.sql']); + } finally { + fs.rmSync(legacy, { recursive: true, force: true }); + } + await pool.query("INSERT INTO users (_id, username) VALUES ('u1', 'ana')"); + await pool.query("INSERT INTO pods (id, name, type, created_by) VALUES ('p1', 'Checkout', 'team', 'u1')"); + await pool.query("INSERT INTO messages (pod_id, user_id, content) VALUES ('p1', 'u1', 'kept')"); + + expect(await runPgMigrations(pool)).toEqual(['0002_record_chains.sql']); + expect((await pool.query('SELECT content FROM messages')).rows).toEqual([{ content: 'kept' }]); + expect((await pool.query('SELECT version FROM schema_migrations ORDER BY version')).rows.map((r) => r.version)) + .toEqual(expect.arrayContaining([expect.stringMatching(/^0001/), expect.stringMatching(/^0002/)])); + expect(await appendRecord({ podId: POD, taskId: 'TASK-1' }, 'e', [{ kind: 'task.claimed', actor: AGENT }], pool)) + .toEqual({ appended: 1, duplicate: false }); + }); + + describe('on the migrated schema', () => { + beforeEach(async () => { + await runPgMigrations(pool); + }); + + it('gives concurrent writers on one chain consecutive seqs, and the export verifies', async () => { + const ref = { podId: POD, taskId: 'TASK-2' }; + const results = await Promise.all(Array.from({ length: 30 }, (_, index) => appendRecord(ref, `event-${index}`, [ + { kind: 'task.claimed', actor: AGENT, data: { leaseUntil: '2026-10-26T05:30:00.000Z' } }, + ], pool))); + expect(results.every((r) => r.appended === 1)).toBe(true); + const out = await readRecordExport(ref, pool); + const verified = verifyRecord(out.jsonl); + expect(verified).toMatchObject({ ok: true, entries: 30 }); + expect(out.head).toBe(verified.head); + expect(kinds(out.jsonl).map((entry) => entry.seq)).toEqual(Array.from({ length: 30 }, (_, i) => i + 1)); + const chain = (await pool.query('SELECT head_seq, head_hash FROM record_chains')).rows; + expect(chain).toEqual([{ head_seq: 30, head_hash: verified.head }]); + }); + + it('creates a new chain once when its first writers race', async () => { + // Both racers can pass the chain-row insert's conflict check before + // either commits; one round rarely shows it, forty almost always do. + const results = []; + for (let round = 0; round < 40; round += 1) { + const ref = { podId: POD, taskId: `TASK-race-${round}` }; + // eslint-disable-next-line no-await-in-loop -- each round is its own race + results.push(...await Promise.allSettled(Array.from({ length: 12 }, (_, index) => appendRecord(ref, `e-${index}`, [ + { kind: 'task.claimed', actor: AGENT }, + ], pool)))); + } + const failures = results.filter((result) => result.status === 'rejected'); + expect(failures.map((result) => result.reason.message)).toEqual([]); + const chains = (await pool.query('SELECT head_seq FROM record_chains')).rows; + expect(chains).toHaveLength(40); + expect(chains.every((row) => row.head_seq === 12)).toBe(true); + }); + + it('refuses a run id that already names another chain, and writes nothing', async () => { + const build = () => ({ lines: ['{}'], headHash: '0'.repeat(64) }); + await appendToChain({ podId: POD, taskId: 'TASK-a' }, 'srv-shared', 'e-1', build, pool); + await expect(appendToChain({ podId: POD, taskId: 'TASK-b' }, 'srv-shared', 'e-1', build, pool)) + .rejects.toThrow('run_id srv-shared belongs to another chain'); + expect((await pool.query('SELECT task_id FROM record_chains')).rows).toEqual([{ task_id: 'TASK-a' }]); + }); + + it('keeps two chains apart when their writes interleave', async () => { + const a = { podId: POD, taskId: 'TASK-3' }; + const b = { podId: POD }; + await Promise.all(Array.from({ length: 10 }, (_, index) => appendRecord(index % 2 ? a : b, `e-${index}`, [ + { kind: index % 2 ? 'task.claimed' : 'tool.call', actor: AGENT, data: index % 2 ? {} : { toolCallId: `t${index}`, title: 'x' } }, + ], pool))); + const [ta, tb] = await Promise.all([readRecordExport(a, pool), readRecordExport(b, pool)]); + expect(verifyRecord(ta.jsonl)).toMatchObject({ ok: true, entries: 5 }); + expect(verifyRecord(tb.jsonl)).toMatchObject({ ok: true, entries: 5 }); + expect(kinds(tb.jsonl).every((entry) => entry.taskId === undefined && entry.podId === POD)).toBe(true); + expect(ta.runId).not.toBe(tb.runId); + }); + + it('writes a redelivered event once, even when the deliveries race', async () => { + const ref = { podId: POD, taskId: 'TASK-4' }; + // On a fresh chain the chain-row insert alone serializes the racers; + // an existing chain is where only the row lock stands between them. + await appendRecord(ref, 'seed', [{ kind: 'task.claimed', actor: AGENT }], pool); + const results = await Promise.all(Array.from({ length: 8 }, () => appendRecord(ref, 'the-one-event', [ + { kind: 'task.claimed', actor: AGENT }, + { kind: 'task.done', actor: BOARD_ACTOR }, + ], pool))); + expect(results.filter((r) => r.appended === 2)).toHaveLength(1); + expect(results.filter((r) => r.duplicate)).toHaveLength(7); + expect(verifyRecord((await readRecordExport(ref, pool)).jsonl)).toMatchObject({ ok: true, entries: 3 }); + }); + + it('rolls back every line of an event when the head update fails, then accepts the retry', async () => { + const ref = { podId: POD, taskId: 'TASK-5' }; + await appendRecord(ref, 'first', [{ kind: 'task.claimed', actor: AGENT }, { kind: 'task.released', actor: AGENT }], pool); + await pool.query(` + CREATE FUNCTION refuse_head() RETURNS trigger AS $$ + BEGIN IF NEW.head_seq > 2 THEN RAISE EXCEPTION 'head update refused'; END IF; RETURN NEW; END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER refuse_head BEFORE UPDATE ON record_chains FOR EACH ROW EXECUTE FUNCTION refuse_head(); + `); + await expect(appendRecord(ref, 'second', [ + { kind: 'task.claimed', actor: AGENT }, + { kind: 'task.done', actor: BOARD_ACTOR }, + ], pool)).rejects.toThrow(/head update refused/); + expect((await pool.query('SELECT count(*)::int AS n FROM record_lines')).rows[0].n).toBe(2); + expect((await pool.query('SELECT head_seq FROM record_chains')).rows[0].head_seq).toBe(2); + + await pool.query('DROP TRIGGER refuse_head ON record_chains'); + expect(await appendRecord(ref, 'second', [ + { kind: 'task.claimed', actor: AGENT }, + { kind: 'task.done', actor: BOARD_ACTOR }, + ], pool)).toEqual({ appended: 2, duplicate: false }); + expect(verifyRecord((await readRecordExport(ref, pool)).jsonl)).toMatchObject({ ok: true, entries: 4 }); + }); + + it('reads an unwritten chain as empty, and a missing store as an error', async () => { + expect(await readRecordExport({ podId: POD, taskId: 'TASK-never' }, pool)) + .toEqual({ runId: null, entries: 0, head: null, jsonl: '' }); + const { readChain, RecordStoreUnavailableError } = require('../../repositories/recordChainRepository'); + await expect(readChain({ podId: POD }, null)).rejects.toBeInstanceOf(RecordStoreUnavailableError); + }); + + it('stores lines byte for byte, so an edited row fails verification', async () => { + const ref = { podId: POD, taskId: 'TASK-6' }; + await appendRecord(ref, 'a', [{ kind: 'task.claimed', actor: AGENT }], pool); + await appendRecord(ref, 'b', [{ kind: 'task.done', actor: BOARD_ACTOR }], pool); + await pool.query("UPDATE record_lines SET line = replace(line, 'task.claimed', 'task.release') WHERE seq = 1"); + const result = verifyRecord((await readRecordExport(ref, pool)).jsonl); + expect(result.ok).toBe(false); + expect(result.errors).toEqual(expect.arrayContaining([expect.objectContaining({ seq: 2, reason: 'prev_mismatch' })])); + }); + }); +}); diff --git a/backend/__tests__/unit/scripts/docReferences.test.js b/backend/__tests__/unit/scripts/docReferences.test.js index 6e1d84caad..96730614ed 100644 --- a/backend/__tests__/unit/scripts/docReferences.test.js +++ b/backend/__tests__/unit/scripts/docReferences.test.js @@ -49,6 +49,15 @@ const DOC_REF = /(? { + const vendored = Object.keys(VENDORED_DOC_ROOTS).find((prefix) => file.startsWith(prefix)); + return vendored ? path.join(REPO_ROOT, VENDORED_DOC_ROOTS[vendored]) : REPO_ROOT; +}; // A root-relative reference (`/docs/….md`) is the repo path with a leading // slash; strip it, or `path.join` is handed an absolute path and resolves the @@ -158,9 +167,17 @@ describe('code does not point at docs that are not there', () => { expect(docs.every((doc) => !fs.existsSync(path.join(REPO_ROOT, doc)))).toBe(true); }); + it('resolves a vendored copy against its source package, not the repo root', () => { + const vendored = references.filter((ref) => ref.file.startsWith('backend/vendor/seats/')); + expect(vendored.length).toBeGreaterThan(0); + expect(new Set(vendored.map((ref) => docRootOf(ref.file)))).toEqual(new Set([path.join(REPO_ROOT, 'packages/seats')])); + expect(vendored.every((ref) => fs.existsSync(path.join(REPO_ROOT, 'packages/seats', ref.doc)))).toBe(true); + expect(docRootOf('backend/services/recordService.ts')).toBe(REPO_ROOT); + }); + it('every referenced doc exists', () => { const missing = [...new Set(references - .filter((ref) => !fs.existsSync(path.join(REPO_ROOT, ref.doc))) + .filter((ref) => !fs.existsSync(path.join(docRootOf(ref.file), ref.doc))) .map((ref) => `${ref.file} -> ${ref.doc}`))]; expect(missing).toEqual([]); }); diff --git a/backend/__tests__/unit/services/recordService.test.js b/backend/__tests__/unit/services/recordService.test.js new file mode 100644 index 0000000000..122aefd921 --- /dev/null +++ b/backend/__tests__/unit/services/recordService.test.js @@ -0,0 +1,157 @@ +/* eslint-disable import/no-unresolved, import/extensions -- TypeScript modules expose CJS exports. */ +// An in-memory chain store with the repository's contract: build() runs with +// the previous line, and a seen event key appends nothing. +const mockChains = new Map(); +jest.mock('../../../repositories/recordChainRepository', () => { + const actual = jest.requireActual('../../../repositories/recordChainRepository'); + const key = (ref) => `${ref.podId}|${ref.taskId || ''}`; + return { + ...actual, + appendToChain: jest.fn(async (ref, runId, eventKey, build) => { + if (!mockChains.has(key(ref))) mockChains.set(key(ref), { runId, lines: [], keys: new Set(), head: null }); + const chain = mockChains.get(key(ref)); + if (chain.keys.has(eventKey)) return { appended: 0, duplicate: true }; + const { lines, headHash } = build(chain.lines[chain.lines.length - 1] || null, chain.runId); + chain.lines.push(...lines); + chain.keys.add(eventKey); + chain.head = headHash; + return { appended: lines.length, duplicate: false }; + }), + readChain: jest.fn(async (ref) => { + const chain = mockChains.get(key(ref)); + return chain ? { runId: chain.runId, headSeq: chain.lines.length, headHash: chain.head, lines: [...chain.lines] } : null; + }), + }; +}); + +const repository = require('../../../repositories/recordChainRepository'); +const record = require('../../../services/recordService'); +const { verifyRecord, hashLine } = require('../../../vendor/seats/src/record/format'); + +const POD = '6a692a1be833c668acdb84cf'; +const AGENT = { _id: '6a920d43b08dee6c42a81c9e', isBot: true, botMetadata: { ownerUserId: '6a47891ee025ce2e0ca1d30b' } }; + +describe('recordService', () => { + const env = process.env.RECORD_V1_PODS; + beforeEach(() => { + mockChains.clear(); + process.env.RECORD_V1_PODS = POD; + }); + afterAll(() => { + if (env === undefined) delete process.env.RECORD_V1_PODS; + else process.env.RECORD_V1_PODS = env; + }); + + it('writes a chain that verifies under the shared reference verifier, and matches the stored head', async () => { + const ref = { podId: POD, taskId: 'TASK-1' }; + await record.appendRecord(ref, 'e1', [{ kind: 'task.claimed', actor: record.agentActor(AGENT) }]); + await record.appendRecord(ref, 'e2', [{ kind: 'task.done', actor: record.BOARD_ACTOR, data: { doneBy: AGENT._id } }]); + const out = await record.readRecordExport(ref); + const result = verifyRecord(out.jsonl); + expect(result).toMatchObject({ ok: true, entries: 2, settled: false }); + expect(out.head).toBe(result.head); + const lines = out.jsonl.trimEnd().split('\n').map((line) => JSON.parse(line)); + expect(lines.map((l) => [l.seq, l.kind, l.podId, l.taskId, l.runId])).toEqual([ + [1, 'task.claimed', POD, 'TASK-1', record.serverRunId(ref)], + [2, 'task.done', POD, 'TASK-1', record.serverRunId(ref)], + ]); + expect(lines[0].actor).toEqual({ type: 'agent', id: AGENT._id, owner: AGENT.botMetadata.ownerUserId }); + expect(lines[1].actor).toEqual({ type: 'system', id: 'board' }); + expect(lines[1].prev).toBe(hashLine(out.jsonl.split('\n')[0])); + }); + + it('a redelivered event key appends nothing', async () => { + const ref = { podId: POD, taskId: 'TASK-2' }; + expect(await record.appendRecord(ref, 'same', [{ kind: 'task.claimed', actor: null }])).toEqual({ appended: 1, duplicate: false }); + expect(await record.appendRecord(ref, 'same', [{ kind: 'task.claimed', actor: null }])).toEqual({ appended: 0, duplicate: true }); + expect((await record.readRecordExport(ref)).entries).toBe(1); + }); + + it('refuses a malformed line before anything is stored', async () => { + const ref = { podId: POD, taskId: 'TASK-3' }; + await expect(record.appendRecord(ref, 'bad', [ + { kind: 'task.claimed', actor: null }, + { kind: 'task.done', actor: record.BOARD_ACTOR, data: { doneBy: 'a display name' } }, + ])).rejects.toThrow(/record:/); + expect((await record.readRecordExport(ref)).entries).toBe(0); + }); + + it('derives one stable, spec-valid run id per chain', () => { + const task = record.serverRunId({ podId: POD, taskId: 'TASK-1' }); + expect(task).toMatch(/^srv-task-[0-9a-f]{32}$/); + expect(record.serverRunId({ podId: POD, taskId: 'TASK-1' })).toBe(task); + expect(record.serverRunId({ podId: POD, taskId: 'TASK-10' })).not.toBe(task); + expect(record.serverRunId({ podId: POD })).toMatch(/^srv-pod-[0-9a-f]{32}$/); + expect(record.serverRunId({ podId: '6a692a1be833c668acdb84ce', taskId: 'TASK-1' })).not.toBe(task); + }); + + it('is on only for listed pods, or every pod with *', () => { + delete process.env.RECORD_V1_PODS; + expect(record.isRecordEnabled(POD)).toBe(false); + process.env.RECORD_V1_PODS = ` other , ${POD} `; + expect(record.isRecordEnabled(POD)).toBe(true); + expect(record.isRecordEnabled('another')).toBe(false); + process.env.RECORD_V1_PODS = '*'; + expect(record.isRecordEnabled('anything')).toBe(true); + }); + + it('takes the actor from the authenticated caller and never guesses one', () => { + expect(record.requestActor({ agentUser: AGENT, agentTokenHash: 'h' })).toEqual({ + type: 'agent', id: AGENT._id, owner: AGENT.botMetadata.ownerUserId, + }); + // A legacy installation token can authenticate without resolving a bot user. + expect(record.requestActor({ agentTokenHash: 'h' })).toBeNull(); + expect(record.requestActor({ userId: '6a47891ee025ce2e0ca1d30b' })).toEqual({ type: 'human', id: '6a47891ee025ce2e0ca1d30b' }); + expect(record.requestActor({ user: { _id: 'has space' } })).toBeNull(); + // An owner that is the agent itself, or not an id, is left off. + expect(record.agentActor({ _id: AGENT._id, botMetadata: { ownerUserId: AGENT._id } })).toEqual({ type: 'agent', id: AGENT._id }); + }); + + describe('emitRecord', () => { + it('writes nothing for a pod that is not enabled', async () => { + repository.appendToChain.mockClear(); + record.emitRecord({ podId: 'not-enabled', taskId: 'TASK-1' }, 'k', [{ kind: 'task.claimed', actor: null }]); + await record.flushRecordWrites(); + expect(repository.appendToChain).not.toHaveBeenCalled(); + }); + + it('keeps one process in source order even when an earlier append is slower', async () => { + const ref = { podId: POD, taskId: 'TASK-4' }; + const real = repository.appendToChain.getMockImplementation(); + repository.appendToChain.mockImplementationOnce(async (...args) => { + await new Promise((resolve) => { setTimeout(resolve, 30); }); + return real(...args); + }); + record.emitRecord(ref, 'first', [{ kind: 'task.claimed', actor: null }]); + record.emitRecord(ref, 'second', [{ kind: 'task.done', actor: record.BOARD_ACTOR }]); + await record.flushRecordWrites(); + const out = await record.readRecordExport(ref); + expect(out.jsonl.trimEnd().split('\n').map((line) => JSON.parse(line).kind)).toEqual(['task.claimed', 'task.done']); + }); + + it('never throws into the source write, and logs one bounded line with no data', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + repository.appendToChain.mockRejectedValueOnce(Object.assign(new Error(`boom ${'x'.repeat(500)}`), { code: 'ECONNREFUSED' })); + expect(() => record.emitRecord({ podId: POD, taskId: 'TASK-5' }, 'k', [ + { kind: 'task.done', actor: record.BOARD_ACTOR, data: { doneBy: 'secret-should-not-log' } }, + ])).not.toThrow(); + await record.flushRecordWrites(); + expect(warn).toHaveBeenCalledTimes(1); + const [message] = warn.mock.calls[0]; + expect(message).toMatch(/^\[record\] append failed pod=6a692a1be833c668acdb84cf task=TASK-5 kinds=task\.done: ECONNREFUSED boom/); + expect(message).not.toMatch(/secret-should-not-log/); + expect(message.length).toBeLessThan(300); + }); + }); + + it('exports OTLP deterministically, with the task as the trace', async () => { + const ref = { podId: POD, taskId: 'TASK-6' }; + await record.appendRecord(ref, 'e1', [{ kind: 'task.claimed', actor: record.agentActor(AGENT) }]); + const { jsonl } = await record.readRecordExport(ref); + const first = JSON.stringify(record.toOtlp(jsonl)); + expect(JSON.stringify(record.toOtlp(jsonl))).toBe(first); + const [span] = record.toOtlp(jsonl).resourceSpans[0].scopeSpans[0].spans; + expect(span.name).toBe('seats.run'); + expect(span.attributes).toEqual(expect.arrayContaining([{ key: 'seats.task.id', value: { stringValue: 'TASK-6' } }])); + }); +}); diff --git a/backend/__tests__/unit/services/taskRecordService.test.js b/backend/__tests__/unit/services/taskRecordService.test.js new file mode 100644 index 0000000000..d65ac6f178 --- /dev/null +++ b/backend/__tests__/unit/services/taskRecordService.test.js @@ -0,0 +1,63 @@ +/* eslint-disable import/no-unresolved, import/extensions -- TypeScript modules expose CJS exports. */ +jest.mock('../../../services/recordService', () => { + const actual = jest.requireActual('../../../services/recordService'); + return { ...actual, emitRecord: jest.fn() }; +}); +const { emitRecord } = require('../../../services/recordService'); +const { taskTransitionLine, recordTaskTransition } = require('../../../services/taskRecordService'); + +const AGENT = { type: 'agent', id: '6a920d43b08dee6c42a81c9e' }; +const AT = new Date('2026-10-26T05:00:00.000Z'); +const LEASE = new Date('2026-10-26T05:30:00.000Z'); +const line = (over) => taskTransitionLine({ podId: 'p', task: { taskId: 'TASK-1' }, at: AT, actor: AGENT, ...over }); + +describe('board transitions → record lines (record-v1 §2, Claim)', () => { + it.each([ + ['a fresh claim', { from: 'pending', to: 'claimed', leaseUntil: LEASE }, + { kind: 'task.claimed', actor: AGENT, data: { leaseUntil: LEASE.toISOString() } }], + // PATCH sets no lease; an older one still on the row is not this claim's. + ['a PATCH to claimed', { from: 'pending', to: 'claimed', task: { claimExpiresAt: LEASE } }, + { kind: 'task.claimed', actor: AGENT, data: {} }], + ['resuming a blocked task', { from: 'blocked', to: 'claimed' }, { kind: 'task.claimed', actor: AGENT, data: {} }], + ['a takeover of a lapsed claim', { from: 'claimed', to: 'claimed', renewal: false }, { kind: 'task.claimed', actor: AGENT, data: {} }], + ['completion', { from: 'claimed', to: 'done', doneBy: 'u1' }, { kind: 'task.done', actor: { type: 'system', id: 'board' }, data: { doneBy: 'u1' } }], + ['completion by the system', { from: null, to: 'done' }, { kind: 'task.done', actor: { type: 'system', id: 'board' }, data: {} }], + ['blocking a claimed task', { from: 'claimed', to: 'blocked' }, { kind: 'task.released', actor: AGENT, data: { reason: 'blocked' } }], + ['a lapsed lease', { from: 'claimed', to: 'pending', reason: 'lease_expired' }, { kind: 'task.released', actor: AGENT, data: { reason: 'lease_expired' } }], + ['returning a claim to pending', { from: 'claimed', to: 'pending' }, { kind: 'task.released', actor: AGENT, data: {} }], + ])('%s', (_label, change, expected) => { + expect(line(change)).toEqual(expected); + }); + + it.each([ + ['a renewal by the holder', { from: 'claimed', to: 'claimed', renewal: true }], + ['a PATCH that leaves a claim claimed', { from: 'claimed', to: 'claimed' }], + ['marking a done task done', { from: 'done', to: 'done' }], + ['unblocking to pending', { from: 'blocked', to: 'pending' }], + ['reopening', { from: 'done', to: 'pending' }], + ['blocking an unclaimed task', { from: 'pending', to: 'blocked' }], + ])('writes nothing for %s', (_label, change) => { + expect(line(change)).toBeNull(); + }); + + it('keys each committed write by the revision it persisted, so a redelivery is one line', () => { + const REVISION = new Date('2026-10-26T05:00:00.001Z'); + recordTaskTransition({ podId: 'p', task: { _id: 'oid', taskId: 'TASK-1', updatedAt: REVISION }, from: 'pending', to: 'claimed', at: AT, actor: AGENT }); + expect(emitRecord).toHaveBeenLastCalledWith({ podId: 'p', taskId: 'TASK-1' }, `task.claimed/oid/${REVISION.toISOString()}`, [expect.objectContaining({ kind: 'task.claimed' })]); + // Two writes in one millisecond still carry two revisions (max(now, previous + 1)). + recordTaskTransition({ podId: 'p', task: { _id: 'oid', taskId: 'TASK-1', updatedAt: new Date(REVISION.getTime() + 1) }, from: 'blocked', to: 'claimed', at: AT, actor: AGENT }); + expect(emitRecord.mock.calls.slice(-2).map((c) => c[1])).toEqual([ + `task.claimed/oid/${REVISION.toISOString()}`, `task.claimed/oid/2026-10-26T05:00:00.002Z`, + ]); + // A writer that bypasses the revision CAS keys on its own commit time. + recordTaskTransition({ podId: 'p', task: { _id: 'oid', taskId: 'TASK-1' }, from: 'claimed', to: 'done', at: AT }); + expect(emitRecord.mock.calls.at(-1)[1]).toBe(`task.done/oid/${AT.toISOString()}`); + }); + + it('needs a pod and a task id', () => { + emitRecord.mockClear(); + recordTaskTransition({ podId: '', task: { taskId: 'TASK-1' }, from: 'pending', to: 'claimed', at: AT }); + recordTaskTransition({ podId: 'p', task: {}, from: 'pending', to: 'claimed', at: AT }); + expect(emitRecord).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/__tests__/unit/vendor/seatsRecordVendor.test.js b/backend/__tests__/unit/vendor/seatsRecordVendor.test.js new file mode 100644 index 0000000000..a709fa73e8 --- /dev/null +++ b/backend/__tests__/unit/vendor/seatsRecordVendor.test.js @@ -0,0 +1,34 @@ +/* eslint-disable import/no-unresolved, import/extensions -- TypeScript modules expose CJS exports. */ +const fs = require('fs'); +const path = require('path'); +const { SOURCE, TARGET, FILES } = require('../../../scripts/sync-seats-record'); + +// One record format, two copies: the backend image cannot see packages/seats. +describe('vendored record-v1 writer', () => { + it.each(FILES)('%s is byte-identical to packages/seats', (file) => { + const vendored = fs.readFileSync(path.join(TARGET, file)); + const source = fs.readFileSync(path.join(SOURCE, file)); + expect(vendored.equals(source)).toBe(true); + }); + + it('copies every module the copied files import', () => { + const missing = FILES.flatMap((file) => { + const text = fs.readFileSync(path.join(SOURCE, file), 'utf8'); + const imports = [...text.matchAll(/^\s*(?:import|export)\s[^'"]*?from\s+['"](\.[^'"]+)['"]/gm)].map((m) => m[1]); + return imports + .map((spec) => path.posix.normalize(path.posix.join(path.posix.dirname(file), spec))) + .filter((target) => !FILES.includes(target)); + }); + expect(missing).toEqual([]); + }); + + it('carries no file the sync list does not name', () => { + const walk = (dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => ( + entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)] + )); + const present = walk(TARGET) + .map((file) => path.relative(TARGET, file).split(path.sep).join('/')) + .filter((file) => file !== 'README.md'); + expect(present.sort()).toEqual([...FILES].sort()); + }); +}); diff --git a/backend/config/migrations/0002_record_chains.sql b/backend/config/migrations/0002_record_chains.sql new file mode 100644 index 0000000000..196af6bae9 --- /dev/null +++ b/backend/config/migrations/0002_record_chains.sql @@ -0,0 +1,32 @@ +-- TASK-513: server-written record-v1 chains (packages/seats/docs/record-v1.md, +-- ADR-033). The backend is the only writer, so one chain per (pod, task) and +-- one taskless chain per pod (task_id = '') stand in for the spec's one run +-- per file. Each line is stored as the exact canonical bytes the shared +-- writer produced, so a reader returns them unchanged and the chain verifies. +CREATE TABLE IF NOT EXISTS record_chains ( + id BIGSERIAL PRIMARY KEY, + pod_id VARCHAR(64) NOT NULL, + task_id VARCHAR(64) NOT NULL DEFAULT '', + run_id VARCHAR(64) NOT NULL UNIQUE, + -- The spec's outside record of the head (section 4): SHA-256 of the last + -- line and the line count. A writer updates both in the transaction that + -- appends the line, under this row's lock. + head_seq INTEGER NOT NULL DEFAULT 0 CHECK (head_seq >= 0), + head_hash CHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (pod_id, task_id) +); + +CREATE TABLE IF NOT EXISTS record_lines ( + chain_id BIGINT NOT NULL REFERENCES record_chains(id) ON DELETE RESTRICT, + seq INTEGER NOT NULL CHECK (seq >= 1), + line TEXT NOT NULL, + -- One source event (a committed board, grant or broker write) can emit + -- several lines; each carries `/`, so a redelivered + -- event is recognised and written once. + source_key VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chain_id, seq), + UNIQUE (chain_id, source_key) +); diff --git a/backend/jest.config.js b/backend/jest.config.js index e0f5efa829..afbff8a7a1 100644 --- a/backend/jest.config.js +++ b/backend/jest.config.js @@ -16,6 +16,9 @@ module.exports = { // parallel workers never race the download lock (the CI flake root cause). globalSetup: '/__tests__/utils/globalSetup.js', transform: { + // The vendored record-v1 closure is ESM (byte-identical to + // packages/seats/src); ts-jest down-levels it to CommonJS like tsc does. + '/vendor/seats/.+\\.js$': ['ts-jest', { tsconfig: { allowJs: true, checkJs: false } }], '^.+\\.tsx?$': ['ts-jest', { tsconfig: { allowJs: true, checkJs: false } }], '^.+\\.jsx?$': 'babel-jest', }, @@ -26,6 +29,7 @@ module.exports = { '!**/coverage/**', '!**/jest.config.js', '!**/__tests__/utils/**', + '!vendor/**', '!server.ts', '!server.js', ], diff --git a/backend/package.json b/backend/package.json index 53d36778c7..e9a22655d0 100644 --- a/backend/package.json +++ b/backend/package.json @@ -34,6 +34,7 @@ "backfill:hosted-mcp-client-id": "ts-node scripts/backfill-hosted-mcp-client-id.ts", "revive:native-installations": "ts-node scripts/revive-native-installations.ts", "set:agent-descriptions": "ts-node scripts/set-agent-descriptions.ts", + "sync:seats-record": "node scripts/sync-seats-record.js", "tsc:check": "tsc --noEmit -p tsconfig.typescheck.json", "tsc:check:all": "tsc --noEmit" }, diff --git a/backend/repositories/recordChainRepository.ts b/backend/repositories/recordChainRepository.ts new file mode 100644 index 0000000000..707841263b --- /dev/null +++ b/backend/repositories/recordChainRepository.ts @@ -0,0 +1,135 @@ +/** + * Storage for server-written record-v1 chains (TASK-513, migration 0002). + * It stores and returns line bytes; it never builds or parses a line. The + * format belongs to the shared writer (vendor/seats/src/record/format.js). + */ + +interface PgClient { + query: (sql: string, params?: unknown[]) => Promise<{ rows: Record[]; rowCount: number | null }>; + release: () => void; +} +interface PgPool { + connect: () => Promise; + query: PgClient['query']; +} + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { pool } = require('../config/db-pg') as { pool: PgPool | null }; + +/** A pod's taskless chain is stored with task_id ''; task ids are never empty. */ +export type ChainRef = { podId: string; taskId?: string }; + +export type StoredChain = { + runId: string; + headSeq: number; + headHash: string | null; + lines: string[]; +}; + +/** Thrown when no PostgreSQL is configured; a reader must not answer "empty". */ +export class RecordStoreUnavailableError extends Error { + constructor() { + super('record store unavailable: PostgreSQL is not configured'); + this.name = 'RecordStoreUnavailableError'; + } +} + +const requirePool = (db: PgPool | null = pool): PgPool => { + if (!db) throw new RecordStoreUnavailableError(); + return db; +}; + +const taskKey = (ref: ChainRef): string => ref.taskId || ''; + +/** + * Append one source event's lines to a chain, in one transaction under the + * chain row's lock, so concurrent writers on any replica get consecutive seqs. + * `build(prevLine, runId)` runs under that lock and returns the new lines and + * the SHA-256 of the last one. A redelivered event key appends nothing. + */ +export async function appendToChain( + ref: ChainRef, + runId: string, + eventKey: string, + build: (prevLine: string | null, runId: string) => { lines: string[]; headHash: string }, + db: PgPool | null = pool, +): Promise<{ appended: number; duplicate: boolean }> { + const client = await requirePool(db).connect(); + try { + await client.query('BEGIN'); + // No conflict target. run_id is unique as well, and a target covers one + // index only, so two first writers racing on one chain could pass the + // (pod_id, task_id) check together and fail on run_id instead. + await client.query( + `INSERT INTO record_chains (pod_id, task_id, run_id) VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING`, + [ref.podId, taskKey(ref), runId], + ); + const chain = (await client.query( + 'SELECT id, run_id, head_seq FROM record_chains WHERE pod_id = $1 AND task_id = $2 FOR UPDATE', + [ref.podId, taskKey(ref)], + )).rows[0] as { id: string; run_id: string; head_seq: number } | undefined; + if (!chain) throw new Error(`record chain not created: run_id ${runId} belongs to another chain`); + const seen = await client.query( + 'SELECT 1 FROM record_lines WHERE chain_id = $1 AND source_key = $2', + [chain.id, `${eventKey}/0`], + ); + if (seen.rowCount) { + await client.query('ROLLBACK'); + return { appended: 0, duplicate: true }; + } + const prevLine = chain.head_seq > 0 + ? (await client.query( + 'SELECT line FROM record_lines WHERE chain_id = $1 AND seq = $2', + [chain.id, chain.head_seq], + )).rows[0].line as string + : null; + const { lines, headHash } = build(prevLine, chain.run_id); + if (lines.length === 0) { + await client.query('ROLLBACK'); + return { appended: 0, duplicate: false }; + } + for (let index = 0; index < lines.length; index += 1) { + // eslint-disable-next-line no-await-in-loop -- one ordered chain, one client + await client.query( + 'INSERT INTO record_lines (chain_id, seq, line, source_key) VALUES ($1, $2, $3, $4)', + [chain.id, chain.head_seq + 1 + index, lines[index], `${eventKey}/${index}`], + ); + } + await client.query( + 'UPDATE record_chains SET head_seq = $2, head_hash = $3, updated_at = NOW() WHERE id = $1', + [chain.id, chain.head_seq + lines.length, headHash], + ); + await client.query('COMMIT'); + return { appended: lines.length, duplicate: false }; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + throw error; + } finally { + client.release(); + } +} + +/** + * Read a chain's lines up to its stored head. Lines and head commit together, + * so bounding by the head read first gives a consistent prefix without a + * transaction. A chain that was never written returns null. + */ +export async function readChain(ref: ChainRef, db: PgPool | null = pool): Promise { + const store = requirePool(db); + const chain = (await store.query( + 'SELECT id, run_id, head_seq, head_hash FROM record_chains WHERE pod_id = $1 AND task_id = $2', + [ref.podId, taskKey(ref)], + )).rows[0] as { id: string; run_id: string; head_seq: number; head_hash: string | null } | undefined; + if (!chain) return null; + const rows = (await store.query( + 'SELECT line FROM record_lines WHERE chain_id = $1 AND seq <= $2 ORDER BY seq', + [chain.id, chain.head_seq], + )).rows as Array<{ line: string }>; + return { + runId: chain.run_id, + headSeq: chain.head_seq, + headHash: chain.head_hash, + lines: rows.map((row) => row.line), + }; +} diff --git a/backend/routes/podRecords.ts b/backend/routes/podRecords.ts new file mode 100644 index 0000000000..579363e743 --- /dev/null +++ b/backend/routes/podRecords.ts @@ -0,0 +1,83 @@ +import express from 'express'; +import rateLimit from 'express-rate-limit'; +import mongoose from 'mongoose'; +import { cloudflareIpRateLimitKeyGenerator } from '../middleware/ipRateLimit'; +import { + isRecordEnabled, readRecordExport, RecordStoreUnavailableError, toOtlp, +} from '../services/recordService'; + +/* eslint-disable @typescript-eslint/no-require-imports, global-require */ +const auth = require('../middleware/dualAuth'); +const Pod = require('../models/Pod'); +const Task = require('../models/Task'); +const DMService = require('../services/dmService'); +/* eslint-enable @typescript-eslint/no-require-imports, global-require */ + +const router = express.Router(); +const recordReadLimit = rateLimit({ + windowMs: 60_000, limit: 120, standardHeaders: 'draft-7', legacyHeaders: false, + keyGenerator: cloudflareIpRateLimitKeyGenerator, +}); + +// Task ids are board tokens (TASK-513); anything else is refused before a read. +const TASK_ID = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/; + +type RecordReq = express.Request & { + userId?: unknown; + user?: { _id?: unknown }; + agentUser?: { _id?: unknown }; +}; + +const notFound = (res: express.Response) => res.status(404).json({ error: 'record_not_found' }); + +/** + * One task's record as record-v1 JSONL, or `?format=otlp` for the spec's + * OTLP/JSON export. + * + * Access follows today's pod access (canViewPod), checked before any record + * byte is loaded, and fails as not-found so the route never confirms a pod or + * task. Lines then go to actual members only: they carry approver and + * credential-owner ids verbatim, and a hash-chained line can't be redacted + * without breaking verification. That is the same set runTraces shows those + * ids to, so admins and agent-DM observers who pass canViewPod get a 403. + */ +router.get('/:podId/record/tasks/:taskId', recordReadLimit, auth, async (req: RecordReq, res) => { + try { + const { podId, taskId } = req.params; + if (!mongoose.isObjectIdOrHexString(podId) || !TASK_ID.test(String(taskId))) return notFound(res); + if (!isRecordEnabled(podId)) return notFound(res); + const userId = req.userId || req.user?._id || req.agentUser?._id; + const pod = await Pod.findById(podId).select('members type').lean(); + if (!pod || !await DMService.canViewPod(userId, pod)) return notFound(res); + // Lean and unpopulated, so members are ObjectIds. + const isMember = (pod.members || []).some((member: unknown) => String(member) === String(userId)); + if (!isMember) return res.status(403).json({ error: 'record_requires_membership' }); + const task = await Task.findOne({ podId: pod._id, taskId }).select('_id').lean(); + if (!task) return notFound(res); + + let record; + try { + record = await readRecordExport({ podId: String(pod._id), taskId: String(taskId) }); + } catch (error) { + // A store outage is an error, never an apparently empty record. + if (!(error instanceof RecordStoreUnavailableError)) { + const err = error as Error; + console.error('[record] read failed:', err.name, String(err.message || '').slice(0, 160)); + } + return res.status(503).json({ error: 'record_store_unavailable' }); + } + res.set('Cache-Control', 'no-store'); + res.set('X-Record-Entries', String(record.entries)); + // The spec's outside record of the head (§4): it covers the last line. + if (record.head) res.set('X-Record-Head', record.head); + if (record.runId) res.set('X-Record-Run-Id', record.runId); + if (req.query.format === 'otlp') return res.json(toOtlp(record.jsonl)); + res.type('application/x-ndjson; charset=utf-8'); + return res.send(record.jsonl); + } catch (error) { + console.error('GET /record error:', (error as Error).name, String((error as Error).message || '').slice(0, 160)); + return res.status(500).json({ error: 'Failed to read record' }); + } +}); + +module.exports = router; diff --git a/backend/routes/tasksApi.ts b/backend/routes/tasksApi.ts index 0144763ac6..6e15f259ef 100644 --- a/backend/routes/tasksApi.ts +++ b/backend/routes/tasksApi.ts @@ -29,6 +29,10 @@ const { resolveAgentDisplayLabel } = require('../services/agentIdentityService') const { recordTaskAttention, resolveTaskAttention } = require('../services/attentionItemService'); // eslint-disable-next-line global-require const { parseAssignee, toSingleLineTitle } = require('../utils/taskFieldNormalization'); +// eslint-disable-next-line global-require +const { requestActor } = require('../services/recordService'); +// eslint-disable-next-line global-require +const { recordTaskTransition } = require('../services/taskRecordService'); /** * Who made this board change, for the agent fan-out. @@ -657,6 +661,16 @@ router.post('/:podId/:taskId/claim', rateLimit({ } const { task, previous: preClaim } = result; emitTaskUpdated(podId, task, 'updated'); + recordTaskTransition({ + podId, + task, + from: preClaim?.status, + to: 'claimed', + at: now, + actor: requestActor(req), + leaseUntil: task.claimExpiresAt, + renewal: preClaim?.status === 'claimed' && preClaim.claimedBy === claimedBy, + }); // Same-holder continuation: the seat that already held (or just lapsed // from) this task re-claimed it. The board UI still updates (socket emit // above); the room is NOT woken — a lease timestamp is not an event. @@ -751,6 +765,7 @@ router.post('/:podId/:taskId/complete', taskWriteRateLimit(30), auth, async (req // tasks completed here. await resolveTaskAttention(task); emitTaskUpdated(podId, task, 'updated'); + recordTaskTransition({ podId, task, from: null, to: 'done', at: now, doneBy: userId }); notifyAgents(req, podId, task, 'updated', { status: 'claimed' }); return res.json({ task }); } catch (err) { @@ -941,6 +956,18 @@ router.post('/:podId/:taskId/updates', taskWriteRateLimit(60), auth, async (req: }, ); leaseRenewed = Boolean(task); + // A note that restores a lapsed claim is a claim, and the record says so. + if (task) { + recordTaskTransition({ + podId, + task, + from: 'pending', + to: 'claimed', + at: now, + actor: requestActor(req), + leaseUntil: task.claimExpiresAt, + }); + } } if (!task) { @@ -1110,6 +1137,17 @@ router.patch('/:podId/:taskId', taskWriteRateLimit(60), auth, async (req: AuthRe if (fieldUpdates.status === 'blocked') await recordTaskAttention(task, { includeBlocked: true }); if (fieldUpdates.status !== undefined && fieldUpdates.status !== 'blocked') await resolveTaskAttention(task); emitTaskUpdated(podId, task, 'updated'); + if (fieldUpdates.status !== undefined && previousTask && writeTime) { + recordTaskTransition({ + podId, + task, + from: previousTask.status, + to: String(fieldUpdates.status), + at: writeTime, + actor: requestActor(req), + doneBy: userId, + }); + } notifyAgents(req, podId, task, 'updated', previousTask); return res.json({ task }); } catch (err) { diff --git a/backend/scripts/sync-seats-record.js b/backend/scripts/sync-seats-record.js new file mode 100644 index 0000000000..266cdcdcfb --- /dev/null +++ b/backend/scripts/sync-seats-record.js @@ -0,0 +1,30 @@ +/** + * Copy the shared record-v1 writer from packages/seats into backend/vendor, + * byte for byte. The backend image is built from backend/ alone, so it cannot + * reach packages/seats at runtime; this keeps one implementation in two + * places, and __tests__/unit/vendor/seatsRecordVendor.test.js fails when they + * differ. Edit packages/seats, then run `npm run sync:seats-record`. + */ +const fs = require('fs'); +const path = require('path'); + +const SOURCE = path.join(__dirname, '../../packages/seats'); +const TARGET = path.join(__dirname, '../vendor/seats'); +// format.js and otel.js plus everything they import, relative to the package. +const FILES = [ + 'src/failures.js', + 'src/harness-plan.js', + 'src/upstream-refusal.js', + 'src/record/format.js', + 'src/record/otel.js', +]; + +if (require.main === module) { + FILES.forEach((file) => { + fs.mkdirSync(path.dirname(path.join(TARGET, file)), { recursive: true }); + fs.copyFileSync(path.join(SOURCE, file), path.join(TARGET, file)); + }); + console.log(`synced ${FILES.length} files into ${path.relative(process.cwd(), TARGET)}`); +} + +module.exports = { SOURCE, TARGET, FILES }; diff --git a/backend/server.ts b/backend/server.ts index 58c4ac624c..8731615859 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -228,6 +228,7 @@ app.use('/api', attributionRoutes); // Before podRoutes: its `/:type/:id` catch-all would answer `/:podId/grants` as type=, id='grants' (Wren 67721). app.use('/api/pods', require('./routes/grants').podGrantsRouter); // tools plan §6: GET /api/pods/:podId/grants, the page's list app.use('/api/pods', require('./routes/runTraces')); +app.use('/api/pods', require('./routes/podRecords')); // TASK-513: GET /api/pods/:podId/record/tasks/:taskId app.use('/api/pods', podRoutes); app.use('/api/billing', require('./routes/billing')); app.use('/api/messages', messageRoutes); diff --git a/backend/services/githubGigService.ts b/backend/services/githubGigService.ts index e696ee7b46..b90f5d5f05 100644 --- a/backend/services/githubGigService.ts +++ b/backend/services/githubGigService.ts @@ -10,6 +10,8 @@ const AgentIdentityService = require('./agentIdentityService'); const AgentMessageService = require('./agentMessageService'); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { emitTaskUpdated } = require('./taskEventService'); +// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports +const { recordTaskTransition } = require('./taskRecordService'); type Element = | { type: 'Text'; text: string } @@ -94,6 +96,7 @@ export const completeTaskForMergedPull = async (podId: string, href: string): Pr ); if (!task) return; emitTaskUpdated(podId, task, 'updated'); + recordTaskTransition({ podId, task, from: null, to: 'done', at }); } }; diff --git a/backend/services/kernelWorkSweepService.ts b/backend/services/kernelWorkSweepService.ts index 98bbeb278d..684670a1a8 100644 --- a/backend/services/kernelWorkSweepService.ts +++ b/backend/services/kernelWorkSweepService.ts @@ -44,6 +44,11 @@ const User = require('../models/User'); // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require const { deriveAgentState } = require('./agentStateService'); +// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require +const { BOARD_ACTOR } = require('./recordService'); +// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require +const { recordTaskTransition } = require('./taskRecordService'); + const KERNEL_ACTOR = 'kernel-sweep'; const TASK_CLAIM_LEASE_MS = 30 * 60 * 1000; @@ -281,7 +286,18 @@ class KernelWorkSweepService { }, { new: true }, ) as TaskRow | null; - if (won) rescued.push(won); + if (won) { + rescued.push(won); + recordTaskTransition({ + podId: t.podId, + task: won, + from: 'claimed', + to: 'pending', + at: now, + actor: BOARD_ACTOR, + reason: 'lease_expired', + }); + } } return { rescued, deferred }; } diff --git a/backend/services/recordService.ts b/backend/services/recordService.ts new file mode 100644 index 0000000000..999f067136 --- /dev/null +++ b/backend/services/recordService.ts @@ -0,0 +1,195 @@ +/** + * The server's record-v1 writer and reader (TASK-513, ADR-033). + * + * Lines are built by the shared reference writer, vendored byte-identical + * from packages/seats/src (see vendor/seats/README.md), so a server line and + * a local seat's line are the same format. The backend is the only writer: + * one chain per (pod, task), plus one taskless chain per pod, each sequenced + * under its PG row lock (repositories/recordChainRepository). + * + * D2: a writer emits only after its source write commits, and a failed emit + * never blocks or undoes that write. Readers get an error, never an empty + * record, when storage is unavailable. + */ +import { createHash } from 'crypto'; +import { + appendToChain, readChain, RecordStoreUnavailableError, type ChainRef, +} from '../repositories/recordChainRepository'; + +type Store = Parameters[1]; + +/* eslint-disable @typescript-eslint/no-require-imports, global-require */ +const format = require('../vendor/seats/src/record/format') as { + appendEntry: (prevLine: string | null, fields: Record) => string; + hashLine: (line: string) => string; + isUserId: (value: unknown) => boolean; + readRecord: (text: string) => Record[]; + BOARD_ACTOR: RecordActor; +}; +const otel = require('../vendor/seats/src/record/otel') as { + toOtlpJson: (entries: Record[], options: { serviceName: string }) => unknown; +}; +/* eslint-enable @typescript-eslint/no-require-imports, global-require */ + +export type RecordActor = { + type: 'agent' | 'human' | 'policy' | 'system'; + id: string; + owner?: string; +}; +export type RecordLineInput = { + kind: string; + /** Omit for the kind's default actor; null records "unknown" (spec §1). */ + actor?: RecordActor | null; + data?: Record; +}; +export { RecordStoreUnavailableError, type ChainRef }; + +export const BOARD_ACTOR: RecordActor = format.BOARD_ACTOR; + +/** + * Pods the record is written and served for. Nothing is public until a + * non-author walks it (TASK-513), so the default is no pod; `*` is every pod. + */ +export const isRecordEnabled = (podId: unknown): boolean => { + const list = String(process.env.RECORD_V1_PODS || '').split(',').map((id) => id.trim()).filter(Boolean); + return list.includes('*') || (typeof podId === 'string' && list.includes(podId)); +}; + +/** + * The spec's runId for a server chain (ADR-033 §8 q3, option a). Derived, so + * the same pod and task always name the same chain, and so the id carries no + * pod or task text beyond what the line's own podId/taskId already say. + */ +export const serverRunId = (ref: ChainRef): string => { + const scope = ref.taskId ? `task/${ref.podId}/${ref.taskId}` : `pod/${ref.podId}`; + const digest = createHash('sha256').update(`commonly.record/${scope}`, 'utf8').digest('hex').slice(0, 32); + return `srv-${ref.taskId ? 'task' : 'pod'}-${digest}`; +}; + +type ActorSource = { + _id?: unknown; + isBot?: boolean; + botMetadata?: { ownerUserId?: unknown } | null; +} | null | undefined; + +/** An agent actor: its user id, plus its owner when that is a valid id. */ +export const agentActor = (user: ActorSource): RecordActor | null => { + const id = user?._id ? String(user._id) : ''; + if (!format.isUserId(id)) return null; + const owner = user?.botMetadata?.ownerUserId ? String(user.botMetadata.ownerUserId) : ''; + return { type: 'agent', id, ...(format.isUserId(owner) && owner !== id ? { owner } : {}) }; +}; + +export const humanActor = (userId: unknown): RecordActor | null => { + const id = userId ? String(userId) : ''; + return format.isUserId(id) ? { type: 'human', id } : null; +}; + +/** + * The authenticated caller of a request as an actor. An agent token sets + * req.agentUser (never req.user); a legacy installation token may not resolve + * one, and then the actor is unknown rather than guessed. + */ +export const requestActor = (req: { + agentUser?: ActorSource; + agentTokenHash?: unknown; + user?: ActorSource; + userId?: unknown; +}): RecordActor | null => { + if (req.agentUser) return agentActor(req.agentUser); + if (req.agentTokenHash) return null; + const user = req.user || null; + if (user?.isBot) return agentActor(user); + return humanActor(req.userId || user?._id); +}; + +/** + * Append one source event's lines now. Throws on a malformed line or an + * unavailable store; callers that must not fail use emitRecord. + */ +export async function appendRecord( + ref: ChainRef, + eventKey: string, + inputs: RecordLineInput[], + db?: Store, +): Promise<{ appended: number; duplicate: boolean }> { + const runId = serverRunId(ref); + return appendToChain(ref, runId, eventKey, (prevLine, storedRunId) => { + const lines: string[] = []; + let prev = prevLine; + for (const input of inputs) { + prev = format.appendEntry(prev, { + runId: storedRunId, + ...(ref.taskId ? { taskId: ref.taskId } : {}), + podId: ref.podId, + ts: new Date().toISOString(), + kind: input.kind, + ...(input.actor === undefined ? {} : { actor: input.actor }), + data: input.data || {}, + }); + lines.push(prev); + } + return { lines, headHash: prev ? format.hashLine(prev) : '' }; + }, db); +} + +// One promise tail per chain keeps this process's emits in source-commit +// order without making the source wait. Across replicas the row lock orders. +const tails = new Map>(); +const chainKey = (ref: ChainRef): string => `${ref.podId}\u0000${ref.taskId || ''}`; + +/** + * Record a committed source event, best-effort. Returns at once; never throws. + * A failure logs one bounded line naming the chain and kinds, never data. + */ +export const emitRecord = (ref: ChainRef, eventKey: string, inputs: RecordLineInput[]): void => { + if (!ref?.podId || inputs.length === 0 || !isRecordEnabled(ref.podId)) return; + const key = chainKey(ref); + const run = async (): Promise => { + try { + await appendRecord(ref, eventKey, inputs); + } catch (error) { + const err = error as { name?: string; code?: string; message?: string }; + console.warn( + `[record] append failed pod=${ref.podId} task=${ref.taskId || '-'} ` + + `kinds=${inputs.map((input) => input.kind).join(',')}: ` + + `${err.code || err.name || 'Error'} ${String(err.message || '').slice(0, 160)}`, + ); + } + }; + const next = (tails.get(key) || Promise.resolve()).then(run); + tails.set(key, next); + void next.then(() => { if (tails.get(key) === next) tails.delete(key); }); +}; + +/** Resolves once every emit queued so far has settled. Tests and shutdown. */ +export const flushRecordWrites = async (): Promise => { + while (tails.size > 0) { + // eslint-disable-next-line no-await-in-loop + await Promise.all([...tails.values()]); + } +}; + +export type RecordExport = { + runId: string | null; + entries: number; + head: string | null; + jsonl: string; +}; + +/** A chain as stored, in record-v1's file form: one line per entry, each ending in \n. */ +export async function readRecordExport(ref: ChainRef, db?: Store): Promise { + const chain = await readChain(ref, db); + if (!chain) return { runId: null, entries: 0, head: null, jsonl: '' }; + return { + runId: chain.runId, + entries: chain.lines.length, + head: chain.headHash, + jsonl: chain.lines.map((line) => `${line}\n`).join(''), + }; +} + +/** The spec's deterministic OTLP/JSON export (§5) of a JSONL record. */ +export const toOtlp = (jsonl: string): unknown => ( + otel.toOtlpJson(format.readRecord(jsonl), { serviceName: 'commonly' }) +); diff --git a/backend/services/starterTaskService.ts b/backend/services/starterTaskService.ts index 305965d714..d7b66256cc 100644 --- a/backend/services/starterTaskService.ts +++ b/backend/services/starterTaskService.ts @@ -24,6 +24,7 @@ import mongoose from 'mongoose'; import Task from '../models/Task'; import { humansAnsweredBy } from './agentAnswerService'; +import { recordTaskTransition } from './taskRecordService'; // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { ESCALATION_MARKERS } = require('./connectorRelayPolicy'); @@ -53,7 +54,10 @@ const completeStarterTask = async (podId: string, sourceRef: string, text: strin }, { new: true }, ); - if (task) emitTaskUpdated(podId, task, 'updated'); + if (task) { + emitTaskUpdated(podId, task, 'updated'); + recordTaskTransition({ podId, task, from: null, to: 'done', at: task.completedAt || new Date() }); + } }; export const completeConnectAgentStarterTask = async ({ diff --git a/backend/services/taskRecordService.ts b/backend/services/taskRecordService.ts new file mode 100644 index 0000000000..ea7ae85535 --- /dev/null +++ b/backend/services/taskRecordService.ts @@ -0,0 +1,70 @@ +/** + * The board's record lines (record-v1 §2, Claim; ADR-033 D4). The board is + * the source of truth: these lines mirror a status change that has already + * committed, and they never move the board. + */ +import { BOARD_ACTOR, emitRecord, type RecordActor, type RecordLineInput } from './recordService'; + +export type TaskStatus = 'pending' | 'claimed' | 'done' | 'blocked' | string; + +export type TaskTransition = { + podId: unknown; + task: { _id?: unknown; taskId?: unknown; updatedAt?: Date | null } | null | undefined; + from: TaskStatus | null | undefined; + to: TaskStatus; + /** + * When the source write committed. The event key prefers the row's own + * `updatedAt`, which the revision CAS persists and advances on every write + * (`max(now, previous + 1)`), so two transitions never share a key and a + * redelivery of one always does. `at` keys writers that bypass the CAS. + */ + at: Date; + /** + * task.claimed: the lease this write set. Only a writer that sets a lease + * (a claim, or a note that restores one) passes it. A PATCH to `claimed` + * sets none, and the row's older lease must not be reported as this one's. + */ + leaseUntil?: Date | null; + /** Who made the change. Unused for task.done, whose actor is the board. */ + actor?: RecordActor | null; + /** task.done: the user who completed it, when a user did. */ + doneBy?: unknown; + /** task.released: why, when the caller knows (`lease_expired`). */ + reason?: string; + /** + * The claim route knows whether the holder already held the claim: then the + * claim only renews a lease, which is not news. Other writers leave it unset, + * and a claimed row that stays claimed counts as no change. + */ + renewal?: boolean; +}; + +/** The record line a board transition writes, or null when it writes none. */ +export const taskTransitionLine = (change: TaskTransition): RecordLineInput | null => { + const { from, to } = change; + if (to === 'claimed' && !(change.renewal ?? from === 'claimed')) { + const leaseUntil = change.leaseUntil instanceof Date ? change.leaseUntil.toISOString() : undefined; + return { kind: 'task.claimed', actor: change.actor ?? null, data: leaseUntil ? { leaseUntil } : {} }; + } + if (to === 'done' && from !== 'done') { + const doneBy = change.doneBy ? String(change.doneBy) : undefined; + return { kind: 'task.done', actor: BOARD_ACTOR, data: doneBy ? { doneBy } : {} }; + } + if (from === 'claimed' && (to === 'blocked' || to === 'pending')) { + const reason = to === 'blocked' ? 'blocked' : change.reason; + return { kind: 'task.released', actor: change.actor ?? null, data: reason ? { reason } : {} }; + } + return null; +}; + +/** Emit the line for a committed board transition, if it has one. Never throws. */ +export const recordTaskTransition = (change: TaskTransition): void => { + const podId = change.podId ? String(change.podId) : ''; + const taskId = change.task?.taskId ? String(change.task.taskId) : ''; + if (!podId || !taskId) return; + const line = taskTransitionLine(change); + if (!line) return; + const source = change.task?._id ? String(change.task._id) : taskId; + const revision = change.task?.updatedAt instanceof Date ? change.task.updatedAt : change.at; + emitRecord({ podId, taskId }, `${line.kind}/${source}/${revision.toISOString()}`, [line]); +}; diff --git a/backend/vendor/seats/README.md b/backend/vendor/seats/README.md new file mode 100644 index 0000000000..f54ef8c1cf --- /dev/null +++ b/backend/vendor/seats/README.md @@ -0,0 +1,14 @@ +# Vendored: the record-v1 reference writer + +These files are copied byte for byte from `packages/seats/src` so the backend +writes the same `seats.record` format as a local seat (ADR-033 D1, TASK-513). +The backend image is built from `backend/` alone and cannot reach +`packages/seats` at runtime. + +Do not edit them here. Change `packages/seats`, then run +`npm run sync:seats-record` in `backend/`. +`__tests__/unit/vendor/seatsRecordVendor.test.js` fails when a copy differs +from its source, or when a copied file imports one that was not copied. + +They are ESM. `tsc` and `ts-jest` (see `jest.config.js`) compile them to +CommonJS with the rest of the backend. diff --git a/backend/vendor/seats/src/failures.js b/backend/vendor/seats/src/failures.js new file mode 100644 index 0000000000..b3ff12fb1d --- /dev/null +++ b/backend/vendor/seats/src/failures.js @@ -0,0 +1,44 @@ +/** + * Named run failures. Every way a run can end badly has one kind, so a trace, + * a retry policy and a dashboard all read the same word. A kind is a contract: + * add one here, never mint one at a call site. + */ +import { scrubAdapterFailure } from './upstream-refusal.js'; + +export const FAILURE_KINDS = Object.freeze({ + stuck: { retryable: true, meaning: 'no progress signal within the liveness window' }, + timeout: { retryable: true, meaning: 'the run exceeded its wall-clock budget' }, + exited: { retryable: true, meaning: 'the CLI process ended before the turn settled' }, + protocol: { retryable: false, meaning: 'the CLI broke its wire protocol' }, + refusal: { retryable: false, meaning: 'the model or provider refused; never retried' }, + max_tokens: { retryable: false, meaning: 'the turn hit its token limit' }, + max_turn_requests: { retryable: false, meaning: 'the turn hit its request limit' }, + cancelled: { retryable: false, meaning: 'the run was cancelled' }, + credential_denied: { retryable: false, meaning: 'the run asked for a credential its grant does not allow' }, + unsupported: { retryable: false, meaning: 'the CLI or its version is outside what this adapter supports' }, + // The harness cannot serve anyone right now. Retrying on the same harness is + // pointless; a run config's fallback moves to its next entry (harness-plan.js). + not_authenticated: { retryable: false, meaning: 'the CLI is not signed in, or its sign-in was rejected' }, + quota: { retryable: false, meaning: 'the account behind the CLI is out of quota or credits' }, + provider_unsupported: { retryable: false, meaning: 'the provider does not serve this model to this account' }, + capacity: { retryable: true, meaning: 'every entry the run config allows is at a host cap or held; it may run once one frees' }, +}); + +export class RunFailure extends Error { + constructor(kind, detail, { cause } = {}) { + if (!FAILURE_KINDS[kind]) throw new Error(`unknown failure kind: ${kind}`); + super(detail ? `${kind}: ${detail}` : kind, cause ? { cause } : undefined); + this.name = 'RunFailure'; + this.kind = kind; + this.retryable = FAILURE_KINDS[kind].retryable; + this.detail = detail || ''; + } +} + +/** Build a failure whose detail has known credentials and secret shapes scrubbed. */ +export const runFailure = (kind, text, { credentials = [], limit } = {}) => { + const { detail } = scrubAdapterFailure(String(text ?? ''), { credentials, limit }); + return new RunFailure(kind, detail); +}; + +export const failureKindOf = (error) => (error instanceof RunFailure ? error.kind : null); diff --git a/backend/vendor/seats/src/harness-plan.js b/backend/vendor/seats/src/harness-plan.js new file mode 100644 index 0000000000..c6dfef4885 --- /dev/null +++ b/backend/vendor/seats/src/harness-plan.js @@ -0,0 +1,329 @@ +/** + * Which harness a seat runs a turn on, and what it does when that harness + * cannot serve. + * + * A seat declares a run config: a primary entry {harness, model, effort?} and + * an ordered fallback list of the same shape. `runTurnWithFallback` tries the + * active entry. When the attempt fails with one of FALLBACK_KINDS (the harness + * cannot serve anyone right now: not signed in, out of quota, the provider + * does not take this model), it moves to the next entry in the same turn, so + * the turn is still served. Every other failure is thrown unchanged and no + * switch happens. A content refusal in particular never switches: re-sending + * refused content through another provider is refusal-shopping. + * + * While a fallback is active, the next turn after the failed entry's cooldown + * starts from the primary again. That turn is the probe: if the primary + * answers, the seat is back on it. If it fails with a fallback kind or a + * retryable liveness kind (PROBE_CARRY_KINDS), the same turn carries on down + * the list. Any other failure, an unnamed one included, ends the turn there: + * an adapter's upstream refusal reaches this module as an unnamed error, and + * carrying it on would be the same refusal-shopping. Either way the cooldown + * restarts, so the next turn stays on the seat's own entry. + * + * An entry the host's policy blocks is skipped and never attempted. An entry + * the host cannot take right now (a per-model concurrency cap, a stop file; + * the caller's `acquire`) is skipped for this turn only. If no entry served, + * the turn fails visibly; nothing substitutes a harness the config does not + * list. + * + * Pure: state goes in and comes out as plain data, persisted by the caller, + * and time is passed in. Nothing here reads a file or the clock. + */ +import { RunFailure, failureKindOf } from './failures.js'; + +/** The failure kinds that move a turn to the next entry. Nothing else does. */ +export const FALLBACK_KINDS = Object.freeze(['not_authenticated', 'quota', 'provider_unsupported']); + +/** + * The failure kinds on which a failed probe carries on to the seat's own + * entry: the fallback kinds, and the retryable kinds that say the primary + * never produced an answer. An allowlist, so a kind added later ends a probe + * turn until someone decides otherwise. + */ +export const PROBE_CARRY_KINDS = Object.freeze([...FALLBACK_KINDS, 'stuck', 'timeout', 'exited']); + +/** How long a failed entry rests before a turn probes it again, per kind. */ +export const PROBE_COOLDOWN_MS = Object.freeze({ + quota: 15 * 60 * 1000, + not_authenticated: 30 * 60 * 1000, + provider_unsupported: 60 * 60 * 1000, +}); + +/** + * Why a seat's active entry changed, as `harness.switched` records it: + * a fallback kind, `blocked` (the active entry is blocked by policy), `probe` + * (a probe moved back up the list) and `operator` (`commonly agent use`). + */ +export const SWITCH_REASONS = Object.freeze([...FALLBACK_KINDS, 'blocked', 'probe', 'operator']); + +/** Fallback entries a config may list after its primary. */ +export const MAX_FALLBACK = 5; +/** Switches kept in state.history, newest last. */ +export const HISTORY_LIMIT = 20; + +const HARNESS = /^[a-z][a-z0-9-]{0,31}$/; +// No '@' and no whitespace: '@' separates the effort in the short form. +const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,127}$/; +const EFFORT = /^[a-z]{1,16}$/; +const REASON = /^[A-Za-z0-9][A-Za-z0-9 ._:/#()-]{0,119}$/; + +const fail = (message) => { throw new Error(`harness plan: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); +}; + +/** + * One entry, as an object or the short form `harness/model[@effort]` + * (`claude/claude-sonnet-5-5@high`). The harness is everything before the first '/', + * so a model may itself contain '/' (`opencode/litellm/deepseek-flash-1017`). + */ +export const parseEntry = (value, where = 'entry') => { + let entry = value; + if (typeof value === 'string') { + const slash = value.indexOf('/'); + if (slash < 1) fail(`${where} must read harness/model[@effort], got ${JSON.stringify(value)}`); + const rest = value.slice(slash + 1); + const at = rest.lastIndexOf('@'); + entry = at === -1 + ? { harness: value.slice(0, slash), model: rest } + : { harness: value.slice(0, slash), model: rest.slice(0, at), effort: rest.slice(at + 1) }; + } + if (!isPlainObject(entry)) fail(`${where} must be an object or harness/model[@effort]`); + refuseUnknownKeys(entry, ['harness', 'model', 'effort'], `${where}.`); + if (typeof entry.harness !== 'string' || !HARNESS.test(entry.harness)) fail(`${where}.harness must match ${HARNESS}`); + if (typeof entry.model !== 'string' || !MODEL.test(entry.model)) fail(`${where}.model must match ${MODEL}`); + if (entry.effort !== undefined && (typeof entry.effort !== 'string' || !EFFORT.test(entry.effort))) { + fail(`${where}.effort must match ${EFFORT}, or be absent`); + } + return Object.freeze({ harness: entry.harness, model: entry.model, ...(entry.effort ? { effort: entry.effort } : {}) }); +}; + +export const entryLabel = (entry) => `${entry.harness}/${entry.model}${entry.effort ? `@${entry.effort}` : ''}`; + +/** + * `{harness, model, effort?, fallback?: [entry, …]}` → `{primary, fallback, chain, key}`. + * Refuses unknown fields, a fallback longer than MAX_FALLBACK and a repeated + * entry (an entry listed twice would be retried under another position). + */ +export const validateRunConfig = (config) => { + if (!isPlainObject(config)) fail('run config must be an object'); + refuseUnknownKeys(config, ['harness', 'model', 'effort', 'fallback'], 'config.'); + const primary = parseEntry({ + harness: config.harness, model: config.model, ...(config.effort !== undefined ? { effort: config.effort } : {}), + }, 'config'); + const list = config.fallback ?? []; + if (!Array.isArray(list)) fail('config.fallback must be an array'); + if (list.length > MAX_FALLBACK) fail(`config.fallback lists at most ${MAX_FALLBACK} entries`); + const fallback = list.map((item, index) => parseEntry(item, `config.fallback[${index}]`)); + const chain = [primary, ...fallback]; + const labels = chain.map(entryLabel); + const repeated = labels.find((label, index) => labels.indexOf(label) !== index); + if (repeated) fail(`config lists ${repeated} twice`); + return Object.freeze({ primary, fallback: Object.freeze(fallback), chain: Object.freeze(chain), key: labels.join(' > ') }); +}; + +/** + * Host policy: `{blocked?: [{harness?, model?, reason}], limits?: [{model, maxConcurrent}]}`. + * A blocked rule names a harness, a model or both, and matches an entry when + * every field it names is equal. The reason is a short operator word (a task + * id), recorded with each skip. A limit caps how many turns on this host run + * one model at once; it is enforced by the caller's `acquire`, which this + * module only consults. Absent or empty policy blocks and caps nothing. + */ +export const validatePolicy = (policy) => { + if (policy === undefined || policy === null) return Object.freeze({ blocked: Object.freeze([]), limits: Object.freeze([]) }); + if (!isPlainObject(policy)) fail('policy must be an object'); + refuseUnknownKeys(policy, ['blocked', 'limits'], 'policy.'); + const list = policy.blocked ?? []; + if (!Array.isArray(list)) fail('policy.blocked must be an array'); + const blocked = list.map((rule, index) => { + const where = `policy.blocked[${index}]`; + if (!isPlainObject(rule)) fail(`${where} must be an object`); + refuseUnknownKeys(rule, ['harness', 'model', 'reason'], `${where}.`); + if (rule.harness === undefined && rule.model === undefined) fail(`${where} names a harness, a model or both`); + if (rule.harness !== undefined && (typeof rule.harness !== 'string' || !HARNESS.test(rule.harness))) fail(`${where}.harness must match ${HARNESS}`); + if (rule.model !== undefined && (typeof rule.model !== 'string' || !MODEL.test(rule.model))) fail(`${where}.model must match ${MODEL}`); + if (typeof rule.reason !== 'string' || !REASON.test(rule.reason)) fail(`${where}.reason must match ${REASON}`); + return Object.freeze({ ...rule }); + }); + const capList = policy.limits ?? []; + if (!Array.isArray(capList)) fail('policy.limits must be an array'); + const limits = capList.map((limit, index) => { + const where = `policy.limits[${index}]`; + if (!isPlainObject(limit)) fail(`${where} must be an object`); + refuseUnknownKeys(limit, ['model', 'maxConcurrent'], `${where}.`); + if (typeof limit.model !== 'string' || !MODEL.test(limit.model)) fail(`${where}.model must match ${MODEL}`); + if (!Number.isInteger(limit.maxConcurrent) || limit.maxConcurrent < 1) fail(`${where}.maxConcurrent must be an integer >= 1`); + return Object.freeze({ model: limit.model, maxConcurrent: limit.maxConcurrent }); + }); + const models = limits.map((limit) => limit.model); + const twice = models.find((model, index) => models.indexOf(model) !== index); + if (twice) fail(`policy.limits lists ${twice} twice`); + return Object.freeze({ blocked: Object.freeze(blocked), limits: Object.freeze(limits) }); +}; + +/** The concurrency cap for this entry's model, or null. */ +export const limitFor = (entry, policy) => (policy?.limits || []).find((limit) => limit.model === entry.model) || null; + +/** The rule that blocks this entry, or null. */ +export const blockedBy = (entry, policy) => (policy?.blocked || []).find((rule) => ( + (rule.harness === undefined || rule.harness === entry.harness) + && (rule.model === undefined || rule.model === entry.model) +)) || null; + +/** State for a config that has never run: on the primary, nothing to probe. */ +export const initialState = (plan) => ({ + v: 1, configKey: plan.key, active: 0, since: null, reason: null, nextProbeAt: null, history: [], +}); + +// A persisted probe time as ms. An ISO string (a hand-edited or foreign +// state file) is read as its time; anything unreadable means the probe is due. +const probeTimeOf = (value) => { + if (Number.isFinite(value)) return value; + const parsed = typeof value === 'string' ? Date.parse(value) : NaN; + return Number.isFinite(parsed) ? parsed : null; +}; + +// State written by another version starts over. State for another config +// (the operator switched the seat) starts over on the new primary and keeps +// its history; the operator command records its own switch. +const stateFor = (plan, state) => { + if (!isPlainObject(state) || state.v !== 1) return initialState(plan); + if (state.configKey !== plan.key) { + return { ...initialState(plan), history: Array.isArray(state.history) ? state.history.slice(-HISTORY_LIMIT) : [] }; + } + const active = Number.isInteger(state.active) && state.active >= 0 && state.active < plan.chain.length ? state.active : 0; + return { + ...initialState(plan), ...state, active, nextProbeAt: probeTimeOf(state.nextProbeAt), + history: Array.isArray(state.history) ? state.history : [], + }; +}; + +/** + * The entry indices this turn tries, in order. From the active entry down; + * from the primary when a fallback is active and its probe is due. + */ +export const turnOrder = (plan, state, now) => { + const current = stateFor(plan, state); + const probing = current.active > 0 && (current.nextProbeAt === null || now >= current.nextProbeAt); + const start = probing ? 0 : current.active; + return { probing, order: plan.chain.map((_, index) => index).slice(start) }; +}; + +const iso = (ms) => new Date(ms).toISOString(); + +/** + * Run one turn on the plan. + * + * @param {object} args + * @param {ReturnType} args.plan + * @param {ReturnType} [args.policy] + * @param {object} [args.state] what the last turn returned; absent for a first run + * @param {number} args.now ms since the epoch + * @param {(entry, info: {index, fresh, probing, switch}) => Promise} args.attempt + * runs the turn on one entry. `fresh` is true when the entry is not the one + * the seat was on, so the caller starts a new session for it. `switch` is + * then `{from, to, reason}` (reason in SWITCH_REASONS) for the caller to + * write into that attempt's trace as `harness.switched`; null otherwise. + * @param {(entry, {limit}) => Promise<{release?: Function}|{unavailable: string}>} [args.acquire] + * asked before each attempt; `{unavailable}` skips the entry for this turn + * (its cap is full, the host holds it). `release` runs when the attempt + * settles. Default: every entry is available. + * @param {(error) => string|null} [args.classify] the failure kind; default failureKindOf + * @param {(skip: {entry, reason, blocked}) => void} [args.onSkip] per skipped entry + * @returns {Promise<{result, entry, index, state, attempts}>} + * On failure it rejects with the failure that ended the turn, carrying + * `harnessState` (persist it as on success) and `harnessAttempts`. + */ +export const runTurnWithFallback = async ({ + plan, policy, state, now, attempt, acquire = async () => ({}), classify = failureKindOf, onSkip = () => {}, +}) => { + if (!plan?.chain?.length) fail('runTurnWithFallback needs a validated plan'); + if (!Number.isFinite(now)) fail('runTurnWithFallback needs now'); + const current = stateFor(plan, state); + const { probing, order } = turnOrder(plan, current, now); + const was = plan.chain[current.active]; + const attempts = []; + let lastKind = null; + let probeKind = null; + let lastError = null; + const skip = (entry, reason, blocked) => { + attempts.push({ entry: entryLabel(entry), outcome: blocked ? 'blocked' : 'unavailable', reason }); + try { onSkip({ entry, reason, blocked }); } catch { /* a listener never fails a turn */ } + }; + + const cooldownOf = (kind) => PROBE_COOLDOWN_MS[kind] ?? PROBE_COOLDOWN_MS.quota; + const failWith = (error, next) => { + error.harnessState = next; + error.harnessAttempts = attempts; + return error; + }; + + for (const index of order) { + const entry = plan.chain[index]; + const rule = blockedBy(entry, policy); + if (rule) { skip(entry, rule.reason, true); continue; } + // eslint-disable-next-line no-await-in-loop + const slot = await acquire(entry, { limit: limitFor(entry, policy) }); + if (typeof slot?.unavailable === 'string') { skip(entry, slot.unavailable, false); continue; } + const fresh = index !== current.active; + // Why this attempt is not on the seat's entry: what failed above it, a + // probe moving up, or the entries above it skipped. + let reason = null; + if (fresh) reason = index < current.active ? 'probe' : (lastKind || 'blocked'); + try { + // eslint-disable-next-line no-await-in-loop + const result = await attempt(entry, { + index, fresh, probing, switch: fresh ? { from: was, to: entry, reason } : null, + }); + attempts.push({ entry: entryLabel(entry), outcome: 'ok' }); + const next = { ...current, active: index }; + if (fresh) { + const why = index < current.active ? 'recovered' : reason; + next.since = iso(now); + next.reason = why; + next.history = [...current.history, { at: iso(now), from: entryLabel(was), to: entryLabel(entry), reason: why }] + .slice(-HISTORY_LIMIT); + } + // Back on the primary there is nothing to probe. On a fallback, the + // next probe waits out the cooldown of what failed above it this turn; + // a turn that simply stayed put keeps the probe time it had. + if (index === 0) next.nextProbeAt = null; + else if (probeKind || fresh) next.nextProbeAt = now + cooldownOf(probeKind || lastKind); + return { result, entry, index, attempts, state: next }; + } catch (error) { + const kind = classify(error); + attempts.push({ entry: entryLabel(entry), outcome: 'failed', kind: kind || null }); + const probeFailed = probing && index < current.active; + if (probeFailed && probeKind === null) probeKind = kind || 'unnamed'; + // Not a reason to change harness: the turn fails as it would have with + // no fallback configured, and the seat stays where it was. A probe that + // failed only for want of an answer (PROBE_CARRY_KINDS) carries on to + // the seat's own entry; any other probe failure ends the turn too. + const carries = probeFailed ? PROBE_CARRY_KINDS.includes(kind) : FALLBACK_KINDS.includes(kind); + if (!carries) { + // A probe failed this turn, so the primary rests again whatever ended + // the turn; otherwise the next turn would probe it straight away. + const next = probeKind ? { ...current, nextProbeAt: now + cooldownOf(probeKind) } : current; + throw failWith(error, next); + } + // Only a failure at or below the seat's entry is a reason to move down + // from it; a probe's failures above it are the probe's own (probeKind). + if (FALLBACK_KINDS.includes(kind) && index >= current.active) lastKind = kind; + lastError = error; + } finally { + try { slot?.release?.(); } catch { /* releasing never fails a turn */ } + } + } + // Nothing served. Stay on the entry the seat was on and push the probe out, + // so the next turn does not go straight back to an entry that just failed. + const next = { ...current }; + if (current.active > 0 && (lastKind || probeKind)) next.nextProbeAt = now + cooldownOf(probeKind || lastKind); + const summary = attempts.map((a) => (a.outcome === 'failed' ? `${a.entry} ${a.kind || 'failed'}` : `${a.entry} ${a.outcome} (${a.reason})`)).join(', '); + if (lastError) throw failWith(lastError, next); + // Every entry was skipped: by policy alone it cannot run at all; with any + // entry merely unavailable it may run once a slot frees, and is retryable. + const onlyBlocked = attempts.every((a) => a.outcome === 'blocked'); + throw failWith(new RunFailure(onlyBlocked ? 'unsupported' : 'capacity', `no entry in the run config could run: ${summary}`), next); +}; diff --git a/backend/vendor/seats/src/record/format.js b/backend/vendor/seats/src/record/format.js new file mode 100644 index 0000000000..38d8b8258b --- /dev/null +++ b/backend/vendor/seats/src/record/format.js @@ -0,0 +1,656 @@ +/** + * The run record, format v1: one canonical JSON object per line, each line + * hash-chained to the one before it. docs/record-v1.md is the specification; + * this file is its reference writer and verifier, and nothing here imports + * the store, so another tool can vendor it alone. + */ +import { createHash } from 'node:crypto'; +import { isIP } from 'node:net'; +import { FAILURE_KINDS } from '../failures.js'; +import { SWITCH_REASONS, entryLabel, parseEntry } from '../harness-plan.js'; + +export const RECORD_VERSION = 1; +export const RECORD_SCHEMA = 'seats.record'; + +export const ACTOR_TYPES = Object.freeze(['agent', 'human', 'policy', 'system']); +// Who decides an approval today is the owner of the credential it would use; +// `decision-owner` is reserved for a separately assigned decider. +export const DECIDED_BY_ROLES = Object.freeze(['credential-owner', 'decision-owner', 'agent-owner', 'pod-member', 'policy']); +// A parked tool call is decided by an owner or by policy, never by a pod member +// who owns neither the credential nor the asking agent (ADR-033 D5). +export const PARKED_CALL_ROLES = Object.freeze(DECIDED_BY_ROLES.filter((role) => role !== 'pod-member')); +export const TOOL_SURFACES = Object.freeze(['connector', 'commonly', 'native']); +export const GRANT_ENDS_ON = Object.freeze(['task.done', 'expiresAt']); +export const REVOKE_REASONS = Object.freeze(['task.done', 'expired', 'cascade', 'manual']); +export const WRITE_MODES = Object.freeze(['read', 'write', 'write-with-confirm']); +export const GRANT_TARGET_KINDS = Object.freeze(['pod', 'seat']); +export const OUTCOMES = Object.freeze(['ok', 'failed', 'stuck', 'cancelled']); +export const RESULT_STATUSES = Object.freeze(['done', 'partial', 'blocked', 'failed']); +// Whose request an agent acts on: 'requester' sent the message that started +// the run, 'assigner' assigned its task. Absent when no human asked. +export const ON_BEHALF_ROLES = Object.freeze(['requester', 'assigner']); +// Why the egress proxy refused a request. The first two can leave host and port null. +export const EGRESS_DENIED_REASONS = Object.freeze([ + 'bad_request', 'not_connect', 'ip_literal', 'not_allowed', 'unresolved', 'private_address', 'unreachable', +]); +const EGRESS_TARGETLESS = new Set(['bad_request', 'not_connect']); +export const SANDBOXES = Object.freeze(['seatbelt', 'bwrap', 'none']); +export const SESSION_METHODS = Object.freeze(['new', 'resume', 'load']); +export const RETRY_STATES = Object.freeze(['scheduled', 'started']); +/** + * agent.retry.errorType. The provider error types opencode 2.1.4 reports on a + * retry, read from its error formatter, plus `other` for any type outside + * them. A closed set, so no agent text can ride in this field. + */ +export const RETRY_ERROR_TYPES = Object.freeze([ + 'provider.auth', + 'provider.content-filter', + 'provider.free-limit', + 'provider.internal', + 'provider.invalid-output', + 'provider.invalid-request', + 'provider.no-route', + 'provider.quota', + 'provider.rate-limit', + 'provider.timeout', + 'provider.transport', + 'provider.unknown', + 'provider.unsupported-operation', + 'other', +]); +// Why a requested session was not reopened: the two forms an ACP adapter +// writes. The code is the JSON-RPC error's, an integer by JSON-RPC 2.0. +const SESSION_FALLBACK = /^(?:agent supports neither session\/resume nor session\/load|session\/(?:resume|load) answered error -?\d{1,16})$/; +export const REDACTED = '[redacted]'; + +/** The actors a writer records for its own bookkeeping, and for the board. */ +export const SYSTEM_ACTOR = Object.freeze({ type: 'system', id: 'seats' }); +export const BOARD_ACTOR = Object.freeze({ type: 'system', id: 'board' }); + +/** + * Every v1 kind. `actor` is who a writer records when the emitter names + * nobody: 'run' (the run's own actor), 'system', or derived from the data + * ('issuer', 'decider'). `trace` is the EVENT_TYPES name, when it differs. + */ +export const KINDS = Object.freeze({ + 'task.claimed': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.released': Object.freeze({ group: 'claim', actor: 'run' }), + 'task.done': Object.freeze({ group: 'claim', actor: 'board' }), + 'grant.issued': Object.freeze({ group: 'grant', actor: 'issuer' }), + 'grant.revoked': Object.freeze({ group: 'grant', actor: 'system' }), + 'permission.requested': Object.freeze({ group: 'approval', actor: 'run' }), + 'permission.decided': Object.freeze({ group: 'approval', actor: 'decider' }), + 'decision.ruled': Object.freeze({ group: 'approval', actor: 'decider' }), + 'attempt.started': Object.freeze({ group: 'attempt', actor: 'run', trace: 'run.started' }), + 'session.opened': Object.freeze({ group: 'attempt', actor: 'system' }), + 'attempt.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + 'run.settled': Object.freeze({ group: 'attempt', actor: 'system' }), + // The attempt that follows runs on another harness entry than the seat was on. + 'harness.switched': Object.freeze({ group: 'attempt', actor: 'system' }), + 'tool.call': Object.freeze({ group: 'tool', actor: 'run' }), + 'tool.update': Object.freeze({ group: 'tool', actor: 'run' }), + 'credential.granted': Object.freeze({ group: 'credential', actor: 'system' }), + 'credential.denied': Object.freeze({ group: 'credential', actor: 'system' }), + 'result.reported': Object.freeze({ group: 'result', actor: 'run' }), + 'turn.prompt_sent': Object.freeze({ group: 'activity', actor: 'system' }), + 'agent.message_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.thought_chunk': Object.freeze({ group: 'activity', actor: 'run' }), + 'agent.retry': Object.freeze({ group: 'activity', actor: 'run' }), + plan: Object.freeze({ group: 'activity', actor: 'run' }), + usage: Object.freeze({ group: 'activity', actor: 'run' }), + 'liveness.stalled': Object.freeze({ group: 'activity', actor: 'system' }), + 'liveness.recovery': Object.freeze({ group: 'activity', actor: 'system' }), + 'process.exited': Object.freeze({ group: 'activity', actor: 'system' }), + // The run's confinement, recorded once per spawn, beside its egress. + 'sandbox.applied': Object.freeze({ group: 'sandbox', actor: 'system' }), + 'egress.allowed': Object.freeze({ group: 'egress', actor: 'system' }), + 'egress.denied': Object.freeze({ group: 'egress', actor: 'system' }), +}); + +const TRACE_TO_KIND = new Map(Object.entries(KINDS).filter(([, spec]) => spec.trace).map(([kind, spec]) => [spec.trace, kind])); + +/** Record kind for an EVENT_TYPES name or a kind; null when it is neither. */ +export const kindOfType = (type) => TRACE_TO_KIND.get(type) || (Object.hasOwn(KINDS, type) ? type : null); +/** The EVENT_TYPES name readRun reports for a kind (unknown kinds pass through). */ +export const traceTypeOf = (kind) => KINDS[kind]?.trace || kind; + +const fail = (message) => { throw new Error(`record: ${message}`); }; +const isPlainObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const isText = (value) => typeof value === 'string' && value.length > 0; +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/; +const RUN_ID = /^[A-Za-z0-9_-]{6,64}$/; +// A user id is an opaque token from the instance: no spaces, so a display +// name can't pass for one. Display names go in actor.name. +const USER_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +export const isUserId = (value) => typeof value === 'string' && USER_ID.test(value); +const TOOL_NAME = /^[^\s*?]{1,256}$/; +const RFC3339_UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +const KIND_SHAPE = /^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$/; +const isTimestamp = (value) => typeof value === 'string' && RFC3339_UTC.test(value) && !Number.isNaN(Date.parse(value)); + +/** + * RFC 8785 (JCS) serialization. For the JSON values a record holds, JCS is + * JSON.stringify with object keys sorted by UTF-16 code units, which is what + * Array.prototype.sort does by default. + * + * `strict` also refuses a lone surrogate in any string or key, as RFC 8785 + * requires. Lines are not strict: agent text from a stream cut mid-pair must + * still record, and JSON.stringify writes the lone half as a \u escape. + */ +const canonicalString = (text, strict) => { + if (strict && !text.isWellFormed()) fail('strings must be well-formed UTF-16 (no lone surrogates)'); + return JSON.stringify(text); +}; +const canonical = (value, strict = false) => { + if (value !== null && typeof value === 'object' && typeof value.toJSON === 'function') return canonical(value.toJSON(), strict); + if (value === null) return 'null'; + switch (typeof value) { + case 'string': return canonicalString(value, strict); + case 'boolean': return value ? 'true' : 'false'; + case 'number': + if (!Number.isFinite(value)) fail('numbers must be finite'); + return JSON.stringify(value); + case 'object': + if (Array.isArray(value)) return `[${value.map((item) => (item === undefined ? 'null' : canonical(item, strict))).join(',')}]`; + return `{${Object.keys(value).filter((key) => value[key] !== undefined).sort() + .map((key) => `${canonicalString(key, strict)}:${canonical(value[key], strict)}`).join(',')}}`; + default: return fail(`cannot serialize a ${typeof value}`); + } +}; + +/** The canonical line for an entry, without its trailing newline. */ +export const canonicalLine = (entry) => { + if (!isPlainObject(entry)) fail('an entry is a JSON object'); + return canonical(entry); +}; + +/** Lowercase hex SHA-256 over a line's UTF-8 bytes: what the next `prev` holds. */ +export const hashLine = (line) => createHash('sha256').update(String(line), 'utf8').digest('hex'); + +/** + * A tool call's arguments as a digest, never the values: `sha256:` + the + * lowercase hex SHA-256 of the arguments' RFC 8785 form. The same arguments + * give the same digest wherever they are seen, so a reader can match a + * tool.call to the permission entries that approved it. Returns undefined + * when there are no arguments, or they are not JSON (a function, NaN, a + * string or key holding a lone surrogate): a writer then omits argsDigest. + */ +const ARGS_DIGEST = /^sha256:[0-9a-f]{64}$/; +export const argsDigestOf = (args) => { + if (args === undefined) return undefined; + try { return `sha256:${hashLine(canonical(args, true))}`; } catch { return undefined; } +}; +export const isArgsDigest = (value) => typeof value === 'string' && ARGS_DIGEST.test(value); + +/** null when the actor is well formed, else the reason. */ +const actorProblem = (actor) => { + if (actor === null) return null; + if (!isPlainObject(actor)) return 'actor must be an object or null'; + if (!ACTOR_TYPES.includes(actor.type)) return `actor.type must be one of ${ACTOR_TYPES.join(', ')}`; + if (!isUserId(actor.id)) return 'actor.id must be an id token (a user id for agent and human actors), never a display name'; + if (actor.owner !== undefined && (actor.type !== 'agent' || !isUserId(actor.owner))) return 'actor.owner is a user id, on agent actors only'; + if (actor.name !== undefined && !isText(actor.name)) return 'actor.name must be a non-empty string'; + if (actor.onBehalfOf !== undefined) { + const behalf = actor.onBehalfOf; + if (actor.type !== 'agent') return 'actor.onBehalfOf is on agent actors only'; + if (!isPlainObject(behalf)) return 'actor.onBehalfOf must be {id, role}'; + const extra = Object.keys(behalf).find((key) => key !== 'id' && key !== 'role'); + if (extra !== undefined) return `unknown field actor.onBehalfOf.${extra}`; + if (!isUserId(behalf.id)) return 'actor.onBehalfOf.id must be an id token (a user id), never a display name'; + if (!ON_BEHALF_ROLES.includes(behalf.role)) return `actor.onBehalfOf.role must be one of ${ON_BEHALF_ROLES.join(', ')}`; + } + return null; +}; + +export const validateActor = (actor) => { + const problem = actorProblem(actor); + if (problem) fail(problem); + if (actor === null) return null; + return Object.freeze({ + type: actor.type, + id: actor.id, + ...(actor.owner ? { owner: actor.owner } : {}), + ...(actor.onBehalfOf ? { onBehalfOf: Object.freeze({ id: actor.onBehalfOf.id, role: actor.onBehalfOf.role }) } : {}), + ...(actor.name ? { name: actor.name } : {}), + }); +}; + +const refuseUnknownKeys = (value, allowed, where) => { + for (const key of Object.keys(value)) if (!allowed.includes(key)) fail(`unknown field ${where}${key}`); +}; +const envNames = (names, where) => { + if (!Array.isArray(names) || !names.every((name) => typeof name === 'string' && ENV_NAME.test(name))) { + fail(`${where} must be credential NAMES ([A-Z_][A-Z0-9_]*), never values`); + } + return [...names]; +}; +const optionalText = (value, where) => { + if (value !== undefined && value !== null && !isText(value)) fail(`${where} must be a non-empty string`); +}; +const userIdOrPolicy = (value, where) => { + if (value !== undefined && value !== null && value !== 'policy' && !isUserId(value)) fail(`${where} must be a user id or "policy"`); +}; +// Only the digest form: raw arguments would carry values. +const optionalArgsDigest = (data, kind) => { + if (data.argsDigest !== undefined && !isArgsDigest(data.argsDigest)) fail(`${kind}.argsDigest must be sha256:<64 hex>, a digest of the arguments, never the arguments`); +}; +// A host as the egress proxy traces it: a lowercase DNS name, an IP literal +// (IPv6 without brackets), or [redacted] when it held a run credential. +const DNS_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; +const isEgressHost = (host) => typeof host === 'string' && (host === REDACTED || isIP(host) !== 0 + || (host.length <= 253 && host.split('.').every((label) => DNS_LABEL.test(label)))); +const isPort = (port) => Number.isInteger(port) && port >= 1 && port <= 65535; +const checkEgressTarget = (data, kind, { nullable }) => { + if (data.host === null && data.port === null && nullable) return; + if (data.host === null || data.port === null) { + fail(nullable ? `${kind}: host and port are null together, or both set` : `${kind} needs a host and a port`); + } + if (!isEgressHost(data.host)) fail(`${kind}.host must be a lowercase DNS name, an IP literal or "${REDACTED}"`); + if (!isPort(data.port)) fail(`${kind}.port must be an integer from 1 to 65535`); +}; + +const checkRole = (role, where, roles = DECIDED_BY_ROLES) => { + if (!roles.includes(role)) fail(`${where} must be one of ${roles.join(', ')}`); +}; + +/** A grant's scope, in the shape of a room grant: exact tool names and a connection id. */ +const checkScope = (scope) => { + if (!isPlainObject(scope)) fail('grant.issued needs a scope'); + refuseUnknownKeys(scope, ['connectionId', 'tools', 'writeMode', 'budget', 'audience'], 'grant.issued.scope.'); + optionalText(scope.connectionId, 'grant.issued.scope.connectionId'); + const tools = scope.tools ?? []; + if (!Array.isArray(tools) || !tools.every((name) => typeof name === 'string' && TOOL_NAME.test(name))) { + fail('grant.issued.scope.tools must be exact tool names (no spaces, no * or ?), not patterns'); + } + if (scope.writeMode !== undefined && !WRITE_MODES.includes(scope.writeMode)) fail(`grant.issued.scope.writeMode must be one of ${WRITE_MODES.join(', ')}`); + let budget; + if (scope.budget !== undefined) { + if (!isPlainObject(scope.budget)) fail('grant.issued.scope.budget must be an object'); + refuseUnknownKeys(scope.budget, ['calls', 'windowMs'], 'grant.issued.scope.budget.'); + if (scope.budget.calls !== undefined && !(Number.isInteger(scope.budget.calls) && scope.budget.calls >= 0)) fail('grant.issued.scope.budget.calls must be an integer >= 0'); + if (scope.budget.windowMs !== undefined && !(Number.isInteger(scope.budget.windowMs) && scope.budget.windowMs >= 1)) fail('grant.issued.scope.budget.windowMs must be an integer >= 1'); + budget = { ...scope.budget }; + } + const audience = scope.audience ?? []; + if (!Array.isArray(audience) || !audience.every(isUserId)) fail('grant.issued.scope.audience must be user ids'); + return { + ...(scope.connectionId ? { connectionId: scope.connectionId } : {}), + tools: [...tools], + ...(scope.writeMode ? { writeMode: scope.writeMode } : {}), + ...(budget ? { budget } : {}), + audience: [...audience], + }; +}; + +// Per-kind checks, applied by every writer. They are strictest where a value +// could ride in: a grant's scope, target and binding refuse fields they don't define. +const CHECKS = { + 'grant.issued': (data, envelope) => { + refuseUnknownKeys(data, ['grantId', 'parentGrantId', 'scope', 'target', 'issuedBy', 'expiresAt', 'binding'], 'grant.issued.'); + if (!isText(data.grantId)) fail('grant.issued needs a grantId'); + optionalText(data.parentGrantId, 'grant.issued.parentGrantId'); + const scope = checkScope(data.scope); + let target; + if (data.target !== undefined) { + if (!isPlainObject(data.target)) fail('grant.issued.target must be an object'); + refuseUnknownKeys(data.target, ['kind', 'id'], 'grant.issued.target.'); + if (!GRANT_TARGET_KINDS.includes(data.target.kind) || !isText(data.target.id)) fail('grant.issued.target is {kind: pod|seat, id}'); + target = { kind: data.target.kind, id: data.target.id }; + } + userIdOrPolicy(data.issuedBy, 'grant.issued.issuedBy'); + if (data.expiresAt !== undefined && !isTimestamp(data.expiresAt)) fail('grant.issued.expiresAt must be an RFC 3339 UTC timestamp'); + let binding; + if (data.binding !== undefined) { + if (!isPlainObject(data.binding)) fail('grant.issued.binding must be an object'); + refuseUnknownKeys(data.binding, ['taskId', 'podId', 'endsOn'], 'grant.issued.binding.'); + if (!isText(data.binding.taskId) || !isText(data.binding.podId)) fail('grant.issued.binding needs taskId and podId: task ids repeat across pods'); + if (data.binding.taskId !== envelope.taskId || data.binding.podId !== envelope.podId) fail('grant.issued.binding must name the run\'s own taskId and podId'); + if (!GRANT_ENDS_ON.includes(data.binding.endsOn)) fail(`grant.issued.binding.endsOn must be one of ${GRANT_ENDS_ON.join(', ')}`); + if (data.binding.endsOn === 'expiresAt' && data.expiresAt === undefined) fail('a grant that ends on expiresAt needs an expiresAt'); + binding = { taskId: data.binding.taskId, podId: data.binding.podId, endsOn: data.binding.endsOn }; + } + return { + ...data, + ...(data.parentGrantId === undefined ? {} : { parentGrantId: data.parentGrantId }), + scope, + ...(target ? { target } : {}), + ...(binding ? { binding } : {}), + }; + }, + 'grant.revoked': (data) => { + if (!Array.isArray(data.grantIds) || data.grantIds.length === 0 || !data.grantIds.every(isText)) fail('grant.revoked needs grantIds: a non-empty array'); + if (new Set(data.grantIds).size !== data.grantIds.length) fail('grant.revoked.grantIds repeats an id'); + if (!REVOKE_REASONS.includes(data.reason)) fail(`grant.revoked.reason must be one of ${REVOKE_REASONS.join(', ')}`); + return { ...data, grantIds: [...data.grantIds] }; + }, + 'task.done': (data, envelope) => { + if (!envelope.taskId) fail('task.done needs a run with a taskId'); + userIdOrPolicy(data.doneBy, 'task.done.doneBy'); + return data; + }, + 'permission.requested': (data) => { + if (!isText(data.question)) fail('permission.requested needs a question: the human-readable ask'); + if (!isPlainObject(data.askedOf)) fail('permission.requested needs askedOf: {id, role}'); + refuseUnknownKeys(data.askedOf, ['id', 'role'], 'permission.requested.askedOf.'); + if (!isUserId(data.askedOf.id)) fail('permission.requested.askedOf.id must be an id token, never a display name'); + checkRole(data.askedOf.role, 'permission.requested.askedOf.role', PARKED_CALL_ROLES); + optionalText(data.grantId, 'permission.requested.grantId'); + optionalText(data.decisionCardId, 'permission.requested.decisionCardId'); + optionalArgsDigest(data, 'permission.requested'); + return { ...data, askedOf: { id: data.askedOf.id, role: data.askedOf.role } }; + }, + 'decision.ruled': (data) => { + if (!isText(data.decisionCardId)) fail('decision.ruled needs a decisionCardId'); + const { value } = data; + if (!(typeof value === 'string' || typeof value === 'boolean' || (typeof value === 'number' && Number.isFinite(value)))) { + fail('decision.ruled.value must be a string, number or boolean'); + } + if (data.decidedBy !== 'policy' && !isUserId(data.decidedBy)) fail('decision.ruled.decidedBy must be a user id or "policy"'); + checkRole(data.decidedByRole, 'decision.ruled.decidedByRole'); + return data; + }, + 'permission.decided': (data) => { + optionalArgsDigest(data, 'permission.decided'); + // A rule-decided call never had a person in the loop; a writer says so + // rather than leave a reader to infer it from rule.list. + // A decision always names who made it: no writer may record one nobody + // can be held to. + const out = { ...data }; + if (!('decidedBy' in out) && !('decidedByRole' in out) && out.rule && out.rule.list !== 'ask') { + out.decidedBy = 'policy'; + out.decidedByRole = 'policy'; + } + if (out.decidedBy === undefined || out.decidedBy === null) fail('permission.decided.decidedBy is required: a user id, or "policy"'); + userIdOrPolicy(out.decidedBy, 'permission.decided.decidedBy'); + checkRole(out.decidedByRole, 'permission.decided.decidedByRole', PARKED_CALL_ROLES); + if ((out.decidedBy === 'policy') !== (out.decidedByRole === 'policy')) { + fail('permission.decided.decidedBy is "policy" exactly when decidedByRole is "policy"'); + } + optionalText(out.decisionCardId, 'permission.decided.decisionCardId'); + return out; + }, + 'tool.call': (data) => { + if (data.surface !== undefined && !TOOL_SURFACES.includes(data.surface)) fail(`tool.call.surface must be one of ${TOOL_SURFACES.join(', ')}`); + optionalArgsDigest(data, 'tool.call'); + return data; + }, + 'credential.granted': (data) => { + const names = envNames(data.names ?? [], 'credential.granted.names'); + if (data.injected === undefined) return { ...data, names }; + const injected = envNames(data.injected, 'credential.granted.injected'); + if (!injected.every((name) => names.includes(name))) fail('credential.granted.injected must be a subset of names'); + return { ...data, names, injected }; + }, + 'credential.denied': (data) => { + envNames([data.name], 'credential.denied.name'); + return data; + }, + 'attempt.settled': (data) => { + if (!Number.isInteger(data.attempt) || data.attempt < 1) fail('attempt.settled.attempt must be a positive integer'); + if (!OUTCOMES.includes(data.outcome)) fail(`attempt.settled.outcome must be one of ${OUTCOMES.join(', ')}`); + if (data.failure !== undefined && data.failure !== null) { + const { kind } = data.failure; + if (kind !== null && kind !== undefined && !FAILURE_KINDS[kind]) fail(`attempt.settled.failure.kind ${kind} is not a FAILURE_KINDS key`); + } + return data; + }, + 'session.opened': (data) => { + refuseUnknownKeys(data, ['method', 'sessionId', 'requestedSessionId', 'fallback'], 'session.opened.'); + if (!SESSION_METHODS.includes(data.method)) fail(`session.opened.method must be one of ${SESSION_METHODS.join(', ')}`); + if (!isText(data.sessionId)) fail('session.opened needs a sessionId'); + if ((data.requestedSessionId === undefined) !== (data.fallback === undefined)) { + fail('session.opened.requestedSessionId and fallback come together: a session was asked for and not reopened'); + } + if (data.requestedSessionId !== undefined) { + if (data.method !== 'new') fail('session.opened.requestedSessionId is set only when method is new'); + if (!isText(data.requestedSessionId)) fail('session.opened.requestedSessionId must be a non-empty string'); + if (typeof data.fallback !== 'string' || !SESSION_FALLBACK.test(data.fallback)) { + fail('session.opened.fallback must be "agent supports neither session/resume nor session/load" or "session/ answered error "'); + } + } + return data; + }, + 'harness.switched': (data) => { + refuseUnknownKeys(data, ['from', 'to', 'reason'], 'harness.switched.'); + for (const key of ['from', 'to']) { + let entry = null; + try { entry = typeof data[key] === 'string' ? parseEntry(data[key]) : null; } catch { entry = null; } + // The canonical label only, so one entry is always written one way. + if (!entry || entryLabel(entry) !== data[key]) fail(`harness.switched.${key} must be an entry label harness/model[@effort]`); + } + if (data.from === data.to) fail('harness.switched.from and to must differ'); + if (!SWITCH_REASONS.includes(data.reason)) fail(`harness.switched.reason must be one of ${SWITCH_REASONS.join(', ')}`); + return data; + }, + 'agent.retry': (data) => { + refuseUnknownKeys(data, ['state', 'attempt', 'nextRetryAt', 'status', 'errorType'], 'agent.retry.'); + if (!RETRY_STATES.includes(data.state)) fail(`agent.retry.state must be one of ${RETRY_STATES.join(', ')}`); + // Every other field is absent when unknown, never null. + if ('attempt' in data && !(Number.isInteger(data.attempt) && data.attempt >= 1)) fail('agent.retry.attempt must be an integer >= 1, or absent'); + if ('nextRetryAt' in data && !isTimestamp(data.nextRetryAt)) fail('agent.retry.nextRetryAt must be an RFC 3339 UTC timestamp, or absent'); + if ('status' in data && !(Number.isInteger(data.status) && data.status >= 100 && data.status <= 599)) fail('agent.retry.status must be an integer HTTP status (100-599), or absent'); + if ('errorType' in data && !RETRY_ERROR_TYPES.includes(data.errorType)) fail(`agent.retry.errorType must be one of ${RETRY_ERROR_TYPES.join(', ')}, or absent`); + return data; + }, + usage: (data) => { + for (const key of ['contextTokens', 'contextSize']) { + const value = data[key]; + if (value !== undefined && value !== null && !(Number.isInteger(value) && value >= 0)) fail(`usage.${key} must be an integer >= 0, or null when unknown`); + } + return data; + }, + 'sandbox.applied': (data) => { + refuseUnknownKeys(data, ['sandbox', 'reason'], 'sandbox.applied.'); + if (!SANDBOXES.includes(data.sandbox)) fail(`sandbox.applied.sandbox must be one of ${SANDBOXES.join(', ')}`); + if (data.reason !== undefined) { + if (data.sandbox !== 'none') fail('sandbox.applied.reason is set only when sandbox is none'); + if (!isText(data.reason)) fail('sandbox.applied.reason must be a non-empty string'); + } + return data; + }, + 'egress.allowed': (data) => { + refuseUnknownKeys(data, ['host', 'port'], 'egress.allowed.'); + checkEgressTarget(data, 'egress.allowed', { nullable: false }); + return data; + }, + 'egress.denied': (data) => { + refuseUnknownKeys(data, ['host', 'port', 'reason'], 'egress.denied.'); + if (!EGRESS_DENIED_REASONS.includes(data.reason)) fail(`egress.denied.reason must be one of ${EGRESS_DENIED_REASONS.join(', ')}`); + checkEgressTarget(data, 'egress.denied', { nullable: EGRESS_TARGETLESS.has(data.reason) }); + return data; + }, + 'result.reported': (data) => { + if (!RESULT_STATUSES.includes(data.status)) fail(`result.reported.status must be one of ${RESULT_STATUSES.join(', ')}`); + if (data.refs !== undefined && !(Array.isArray(data.refs) && data.refs.every((ref) => isPlainObject(ref) && isText(ref.type) && isText(ref.id)))) { + fail('result.reported.refs must be [{type, id}]'); + } + return data; + }, +}; + +/** + * Validate one kind's data and return the copy a writer records. `envelope` + * is {taskId, podId}, for checks that tie data to the run it sits in. + */ +export const normalizeData = (kind, data = {}, envelope = {}) => { + if (!isPlainObject(data)) fail(`${kind} data must be an object`); + const check = CHECKS[kind]; + return check ? check({ ...data }, envelope) : { ...data }; +}; + +/** A grant's rules acting on their own: named by the grant when its id is a token. */ +export const policyActorId = (grantId) => (isText(grantId) && isUserId(`grant:${grantId}`) ? `grant:${grantId}` : 'policy'); +const policyActor = (grantId) => ({ type: 'policy', id: policyActorId(grantId) }); + +/** The actor a writer records when the emitter names none. */ +export const defaultActor = (kind, data = {}, runActor = null) => { + switch (KINDS[kind]?.actor) { + case 'run': return runActor; + case 'system': return SYSTEM_ACTOR; + case 'board': return BOARD_ACTOR; + case 'issuer': + if (data.issuedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.issuedBy) ? { type: 'human', id: data.issuedBy } : null; + case 'decider': + if (data.decidedByRole === 'policy' || data.decidedBy === 'policy') return policyActor(data.grantId); + return isUserId(data.decidedBy) ? { type: 'human', id: data.decidedBy } : null; + default: return null; + } +}; + +const parseLine = (line) => { + let entry; + try { entry = JSON.parse(line); } catch { return fail('the previous line is not JSON'); } + if (!isPlainObject(entry) || !Number.isInteger(entry.seq)) fail('the previous line is not a record entry'); + return entry; +}; + +/** + * appendEntry(prevLine, fields, { runActor }?) → the next line, canonical, + * without its newline. + * + * fields: { runId, taskId?, podId?, ts, kind, actor?, data, seq? }. seq and + * prev come from prevLine (null for the first line); a given seq must agree. + * An absent actor takes the kind's default; null records "unknown". + */ +export const appendEntry = (prevLine, fields = {}, { runActor = null } = {}) => { + let seq = 1; + let prev = null; + if (prevLine !== null && prevLine !== undefined) { + const before = parseLine(prevLine); + seq = before.seq + 1; + prev = hashLine(prevLine); + if (before.runId !== fields.runId) fail('one record holds one run'); + if (before.taskId !== fields.taskId || before.podId !== fields.podId) fail('taskId and podId are fixed for the whole run'); + if (before.kind === 'run.settled') fail('nothing is written after run.settled'); + } + if (fields.seq !== undefined && fields.seq !== seq) fail(`seq ${fields.seq} does not follow ${seq - 1}`); + if (!RUN_ID.test(String(fields.runId))) fail('runId must be 6-64 characters of [A-Za-z0-9_-]'); + optionalText(fields.taskId, 'taskId'); + optionalText(fields.podId, 'podId'); + if (fields.taskId && !fields.podId) fail('a taskId needs a podId: task ids repeat across pods'); + const ts = fields.ts instanceof Date ? fields.ts.toISOString() : fields.ts; + if (!isTimestamp(ts)) fail('ts must be an RFC 3339 UTC timestamp'); + if (!Object.hasOwn(KINDS, fields.kind)) fail(`unknown record kind: ${fields.kind}`); + const data = normalizeData(fields.kind, fields.data ?? {}, { taskId: fields.taskId, podId: fields.podId }); + const actor = validateActor(fields.actor === undefined ? defaultActor(fields.kind, data, runActor) : fields.actor); + return canonicalLine({ + v: RECORD_VERSION, + schema: RECORD_SCHEMA, + runId: fields.runId, + ...(fields.taskId ? { taskId: fields.taskId } : {}), + ...(fields.podId ? { podId: fields.podId } : {}), + seq, + ts, + kind: fields.kind, + actor, + data, + prev, + }); +}; + +/** Split a record file's text into lines; a last line with no newline is torn. */ +export const splitRecord = (text) => { + if (!text) return { lines: [], torn: null }; + const parts = String(text).split('\n'); + const tail = parts.pop(); + return { lines: parts, torn: tail === '' ? null : tail }; +}; + +const envelopeProblem = (entry) => { + if (!Number.isInteger(entry.v) || entry.v < 1) return 'v'; + if (entry.schema !== RECORD_SCHEMA) return 'schema'; + if (!RUN_ID.test(String(entry.runId))) return 'runId'; + if (entry.taskId !== undefined && !isText(entry.taskId)) return 'taskId'; + if (entry.podId !== undefined && !isText(entry.podId)) return 'podId'; + if (entry.taskId !== undefined && entry.podId === undefined) return 'podId'; + if (!Number.isInteger(entry.seq) || entry.seq < 1) return 'seq'; + if (!isTimestamp(entry.ts)) return 'ts'; + if (typeof entry.kind !== 'string' || !KIND_SHAPE.test(entry.kind)) return 'kind'; + if (actorProblem(entry.actor)) return 'actor'; + if (!isPlainObject(entry.data)) return 'data'; + if (entry.prev !== null && !(typeof entry.prev === 'string' && /^[0-9a-f]{64}$/.test(entry.prev))) return 'prev'; + return null; +}; + +/** + * verifyRecord(text | lines) → { ok, errors: [{seq, reason, detail?}], entries, head, settled } + * + * Text is a whole file, so a torn final line (no newline) can be seen; an + * array is taken as complete lines. Each problem is reported at the seq of + * the line where it was observed: + * torn the last line was cut short by a crash mid-write + * unparseable not JSON + * bad_envelope an envelope field is missing or malformed (detail names it) + * unsupported_version v is newer than this reader + * not_canonical the bytes are not the canonical form of what they parse to + * seq_mismatch seq is not the previous seq + 1 (a deleted or moved line) + * prev_mismatch prev is not the hash of the line before it: that line, or + * this one's prev, was changed, removed or moved + * run_mismatch runId, taskId or podId differs from the first line + * after_settled a line follows run.settled + * The chain re-syncs after each problem, so one edit is reported once. + */ +export const verifyRecord = (input) => { + const { lines, torn } = Array.isArray(input) ? { lines: input.map(String), torn: null } : splitRecord(input); + const errors = []; + const report = (seq, reason, detail) => errors.push(detail ? { seq, reason, detail } : { seq, reason }); + let expected = 1; + let prevRaw = null; + let first = null; + let settled = false; + for (const line of lines) { + let entry; + try { entry = JSON.parse(line); } catch { entry = undefined; } + if (!isPlainObject(entry)) { + report(expected, 'unparseable'); + prevRaw = line; + expected += 1; + continue; + } + const seq = Number.isInteger(entry.seq) ? entry.seq : expected; + if (Number.isInteger(entry.v) && entry.v > RECORD_VERSION) report(seq, 'unsupported_version', `v${entry.v}`); + const problem = envelopeProblem(entry); + if (problem) report(seq, 'bad_envelope', problem); + let canonicalForm = null; + try { canonicalForm = canonicalLine(entry); } catch { /* reported below */ } + if (canonicalForm !== line) report(seq, 'not_canonical'); + if (seq !== expected) report(seq, 'seq_mismatch', `expected ${expected}`); + const expectedPrev = prevRaw === null ? null : hashLine(prevRaw); + if (entry.prev !== expectedPrev) report(seq, 'prev_mismatch'); + if (first === null) first = entry; + else if (entry.runId !== first.runId || entry.taskId !== first.taskId || entry.podId !== first.podId) report(seq, 'run_mismatch'); + if (settled) report(seq, 'after_settled'); + if (entry.kind === 'run.settled') settled = true; + prevRaw = line; + expected = seq + 1; + } + if (torn !== null) report(expected, 'torn'); + return { + ok: errors.length === 0, + errors, + entries: lines.length, + head: prevRaw === null ? null : hashLine(prevRaw), + settled, + }; +}; + +/** Parse a record's text into entries, skipping a torn tail and unparseable lines. */ +export const readRecord = (text) => splitRecord(text).lines.flatMap((line) => { + try { + const entry = JSON.parse(line); + return isPlainObject(entry) ? [entry] : []; + } catch { return []; } +}); + +/** + * Which surface a tool name belongs to, for an adapter that knows the real + * tool name (an MCP server/tool pair, say). Never apply it to a title: a + * title is the agent's own text and can claim any prefix. + */ +export const toolSurfaceOf = (toolName) => { + const name = String(toolName || ''); + if (/^(?:mcp__commonly__|commonly_)/.test(name)) return 'commonly'; + if (/^mcp__[^_]/.test(name)) return 'connector'; + return 'native'; +}; diff --git a/backend/vendor/seats/src/record/otel.js b/backend/vendor/seats/src/record/otel.js new file mode 100644 index 0000000000..6d59ff26f7 --- /dev/null +++ b/backend/vendor/seats/src/record/otel.js @@ -0,0 +1,238 @@ +/** + * Export run records as OTLP/JSON traces (an ExportTraceServiceRequest), so + * a tracing backend can show who let which agent do what beside its own + * spans. docs/record-v1.md §5 is the mapping; the gen_ai.* names follow the + * OpenTelemetry GenAI conventions, which are Development status upstream. + * + * Ids are derived, never random: the same entries export to the same bytes. + */ +import { createHash } from 'node:crypto'; +import { KINDS, RECORD_SCHEMA, RECORD_VERSION } from './format.js'; + +const SPAN_KIND_INTERNAL = 1; +const STATUS_UNSET = 0; +const STATUS_ERROR = 2; +const FAILED_OUTCOMES = new Set(['failed', 'stuck', 'cancelled']); +// Content stays out of the export, as upstream makes content capture opt-in. +const CONTENT_KINDS = new Set(['agent.message_chunk', 'agent.thought_chunk']); + +const digest = (text, bytes) => { + const hex = createHash('sha256').update(text, 'utf8').digest('hex').slice(0, bytes * 2); + // An all-zero id is invalid in OTLP; vanishingly unlikely, cheap to exclude. + return /^0+$/.test(hex) ? digest(`${text}#`, bytes) : hex; +}; + +/** 16-byte trace id per task (task ids repeat across pods, so the pod is in it), else per run. */ +export const traceIdOf = (entry) => digest(entry.taskId + ? `seats.record/trace/task/${entry.podId}/${entry.taskId}` + : `seats.record/trace/run/${entry.runId}`, 16); +/** 8-byte span id for the span an entry opens; the run span uses seq 0. */ +export const spanIdOf = (runId, seq) => digest(`seats.record/span/${runId}/${seq}`, 8); + +/** RFC 3339 → Unix nanoseconds as a decimal string, keeping sub-millisecond digits. */ +const nanos = (ts) => { + const match = /^(.*T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?Z$/.exec(ts); + if (!match) return '0'; + return String(BigInt(Date.parse(`${match[1]}Z`)) * 1000000n + BigInt((match[2] || '').padEnd(9, '0'))); +}; + +const snake = (key) => key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase(); + +const anyValue = (value) => { + if (typeof value === 'string') return { stringValue: value }; + if (typeof value === 'boolean') return { boolValue: value }; + if (Number.isInteger(value)) return { intValue: String(value) }; + if (typeof value === 'number') return { doubleValue: value }; + if (Array.isArray(value)) return { arrayValue: { values: value.filter((v) => v !== null && v !== undefined).map(anyValue) } }; + return { + kvlistValue: { + values: Object.keys(value).filter((k) => value[k] !== null && value[k] !== undefined).sort() + .map((k) => ({ key: snake(k), value: anyValue(value[k]) })), + }, + }; +}; +const attr = (key, value) => ({ key, value: anyValue(value) }); +const attrs = (pairs) => pairs.filter(([, value]) => value !== null && value !== undefined).map(([key, value]) => attr(key, value)); + +/** Nested data → `seats.` attributes, keys sorted for a stable export. */ +const flatten = (data, prefix = 'seats') => Object.keys(data || {}).sort().flatMap((key) => { + const value = data[key]; + const name = `${prefix}.${snake(key)}`; + if (value === null || value === undefined) return []; + if (typeof value === 'object' && !Array.isArray(value)) return flatten(value, name); + return [attr(name, value)]; +}); + +const eventOf = (entry) => ({ + timeUnixNano: nanos(entry.ts), + name: entry.kind, + attributes: [ + attr('seats.seq', entry.seq), + ...(entry.actor ? attrs([ + ['seats.actor.type', entry.actor.type], ['seats.actor.id', entry.actor.id], + ['seats.actor.owner', entry.actor.owner], ['seats.actor.name', entry.actor.name], + ['seats.actor.on_behalf_of.id', entry.actor.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor.onBehalfOf?.role], + ]) : []), + ...flatten(entry.data), + ], +}); + +const fail = (span, errorType) => { + span.status = { code: STATUS_ERROR, message: errorType }; + span.attributes.push(attr('error.type', errorType)); +}; + +/** One run's entries, in seq order → its spans. */ +const spansOfRun = (entries) => { + const head = entries[0]; + const traceId = traceIdOf(head); + const last = entries[entries.length - 1]; + const span = ({ seq, parent, name, attributes, start }) => ({ + traceId, + spanId: spanIdOf(head.runId, seq), + ...(parent ? { parentSpanId: parent.spanId } : {}), + name, + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: nanos(start), + endTimeUnixNano: null, + attributes, + events: [], + status: { code: STATUS_UNSET }, + }); + const run = span({ + seq: 0, + name: 'seats.run', + start: head.ts, + attributes: attrs([['seats.run.id', head.runId], ['seats.task.id', head.taskId], ['seats.pod.id', head.podId]]), + }); + const spans = [run]; + let attempt = null; // { span, number, tools: Map, usage, context, sessionId } + const cards = new Map(); // decisionCardId → the span its request sits on + const end = (s, ts) => { if (s && s.endTimeUnixNano === null) s.endTimeUnixNano = nanos(ts); }; + const closeAttempt = (ts) => { + if (!attempt) return; + for (const tool of attempt.tools.values()) end(tool, ts); + if (attempt.usage) { + attempt.span.attributes.push(...attrs([ + ['gen_ai.usage.input_tokens', attempt.usage.inputTokens], ['gen_ai.usage.output_tokens', attempt.usage.outputTokens], + ])); + } + attempt.span.attributes.push(...attrs([ + ['seats.usage.context_tokens', attempt.context.tokens], ['seats.usage.context_size', attempt.context.size], + ])); + // The session an attempt opened names the conversation when attempt.started did not. + if (attempt.sessionId && !attempt.span.attributes.some((a) => a.key === 'gen_ai.conversation.id')) { + attempt.span.attributes.push(attr('gen_ai.conversation.id', attempt.sessionId)); + } + end(attempt.span, ts); + attempt = null; + }; + const current = () => attempt?.span || run; + + for (const entry of entries) { + const { kind, data = {} } = entry; + if (!Object.hasOwn(KINDS, kind) || CONTENT_KINDS.has(kind)) continue; // unknown kinds are skipped + if (kind === 'attempt.started') { + closeAttempt(entry.ts); + const agentName = entry.actor?.name || entry.actor?.id; + const s = span({ + seq: entry.seq, + parent: run, + name: agentName ? `invoke_agent ${agentName}` : 'invoke_agent', + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'invoke_agent'], + ['gen_ai.agent.id', entry.actor?.id], ['gen_ai.agent.name', agentName], + ['gen_ai.conversation.id', data.sessionId], + ['seats.agent.owner', entry.actor?.owner], + ['seats.agent.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.agent.on_behalf_of.role', entry.actor?.onBehalfOf?.role], + ['seats.attempt.number', data.attempt], ['seats.adapter', data.adapter], + ['seats.cli.name', data.cli?.name], ['seats.cli.version', data.cli?.version], + ]), + }); + spans.push(s); + attempt = { + span: s, number: data.attempt, agentName, tools: new Map(), usage: null, context: { tokens: null, size: null }, sessionId: null, + }; + } else if (kind === 'attempt.settled' && attempt && attempt.number === data.attempt) { + attempt.span.attributes.push(attr('seats.attempt.outcome', data.outcome)); + if (FAILED_OUTCOMES.has(data.outcome)) fail(attempt.span, data.failure?.kind || data.outcome); + closeAttempt(entry.ts); + } else if (kind === 'run.settled') { + run.attributes.push(...attrs([['seats.run.outcome', data.outcome], ['seats.run.attempts', data.attempts]])); + if (FAILED_OUTCOMES.has(data.outcome)) { + const errorType = data.failure?.kind || data.outcome; + fail(run, errorType); + // The run's outcome is the last attempt's when nothing settled it. + if (attempt) fail(attempt.span, errorType); + } + closeAttempt(entry.ts); + end(run, entry.ts); + } else if (kind === 'tool.call') { + const toolName = data.name || data.title || 'tool'; + const s = span({ + seq: entry.seq, + parent: current(), + name: `execute_tool ${toolName}`, + start: entry.ts, + attributes: attrs([ + ['gen_ai.operation.name', 'execute_tool'], + ['gen_ai.tool.name', toolName], ['gen_ai.tool.call.id', data.toolCallId], + ['gen_ai.agent.name', attempt?.agentName], + ['seats.actor.on_behalf_of.id', entry.actor?.onBehalfOf?.id], ['seats.actor.on_behalf_of.role', entry.actor?.onBehalfOf?.role], + ['seats.tool.kind', data.kind], ['seats.tool.surface', data.surface], ['seats.tool.args_digest', data.argsDigest], + ]), + }); + spans.push(s); + if (attempt && data.toolCallId) attempt.tools.set(data.toolCallId, s); + } else if (kind === 'tool.update') { + const tool = attempt?.tools.get(data.toolCallId); + (tool || current()).events.push(eventOf(entry)); + if (tool && data.status === 'failed') fail(tool, 'tool_failed'); + if (tool && (data.status === 'completed' || data.status === 'failed')) end(tool, entry.ts); + } else if (kind === 'usage' && attempt) { + attempt.usage = data; + // Context fill comes from the stream's usage updates; a later entry without it keeps the last known. + if (Number.isInteger(data.contextTokens)) attempt.context.tokens = data.contextTokens; + if (Number.isInteger(data.contextSize)) attempt.context.size = data.contextSize; + attempt.span.events.push(eventOf(entry)); + } else if (kind === 'permission.requested' || kind === 'permission.decided') { + const target = attempt?.tools.get(data.toolCallId) || current(); + target.events.push(eventOf(entry)); + if (data.decisionCardId && !cards.has(data.decisionCardId)) cards.set(data.decisionCardId, target); + } else if (kind === 'session.opened') { + if (attempt && typeof data.sessionId === 'string') attempt.sessionId = data.sessionId; + current().events.push(eventOf(entry)); + } else if (kind === 'decision.ruled') { + // A ruling on a card that a request rode on joins that request; any other is an event where it happened. + (cards.get(data.decisionCardId) || current()).events.push(eventOf(entry)); + } else { + current().events.push(eventOf(entry)); + } + } + closeAttempt(last.ts); + end(run, last.ts); + return spans; +}; + +/** + * toOtlpJson(entries, { serviceName? }) → ExportTraceServiceRequest (OTLP/JSON) + * + * `entries` are record entries (readRun's output, or parsed lines) from one + * or more runs. Entries of another schema or a newer version are left out. + */ +export const toOtlpJson = (entries, { serviceName = 'seats' } = {}) => { + const runs = new Map(); + for (const entry of entries || []) { + if (!entry || entry.schema !== RECORD_SCHEMA || !(entry.v <= RECORD_VERSION)) continue; + if (!runs.has(entry.runId)) runs.set(entry.runId, []); + runs.get(entry.runId).push(entry); + } + const spans = [...runs.values()].flatMap((run) => spansOfRun([...run].sort((a, b) => a.seq - b.seq))); + return { + resourceSpans: [{ + resource: { attributes: [attr('service.name', serviceName)] }, + scopeSpans: [{ scope: { name: RECORD_SCHEMA, version: String(RECORD_VERSION) }, spans }], + }], + }; +}; diff --git a/backend/vendor/seats/src/upstream-refusal.js b/backend/vendor/seats/src/upstream-refusal.js new file mode 100644 index 0000000000..12f7f30cca --- /dev/null +++ b/backend/vendor/seats/src/upstream-refusal.js @@ -0,0 +1,362 @@ +/** + * TASK-096 — name a refused model route, instead of reporting the silence it + * causes. + * + * Measured 2026-09-22 against pi 0.84.1 with a route answering 429 (the shape + * that made a seat look broken for two and a half days): pi runs its own retry + * ladder, reports the refusal on `message_end`/`auto_retry_end` JSON events, and + * EXITS 0 with no assistant text. `extractReply` in the pi adapter collected only + * `type: "error"` events — of which this produces none — so the wrapper logged + * `no wrapper-post (empty output)` and the cause sat unread in the stream. + * + * Two rules govern the text that reaches a seat log: + * + * 1. KEEP-LIST, not strip-list (Vera 70876, Wren 70877). A strip-list removes + * only the shapes we thought of, and a 401 can echo the key back. So a body + * contributes one NAMED field or nothing at all. + * 2. Exact-match refusal for credentials. If the kept text contains a value + * this spawn was handed, the text is dropped — a known value, not a guessed + * shape (Wren 70877). + * + * Two limits follow from rule 2. They are the accepted cost of refusing guessed + * shapes, named here so they are not rediscovered as defects later (Vera 71082): + * + * a. The match is exact and against THIS spawn's values only, so a credential + * that arrives transformed — base64 inside a proxy error, percent-encoded in + * a URL echo — does not match and the text is kept. Matching transformed + * shapes instead would reintroduce the class rule 1 exists to avoid. + * b. The keep-list can still surface someone ELSE's secret. A third party's key + * relayed in an upstream error is not detectable here, because the one + * inventory this module has is the set of values it was itself handed. + * + * Scope: this reader was wired into the pi adapter only. TASK-103 extended it + * to codex and claude, which are NOT silent the way pi was: the claude adapter + * rejects on a non-zero exit and deliberately reports stdout with it, because a + * `-p` run writes quota conditions there; the codex adapter rejects on + * `turnFailedMessage` even at exit 0. Their gap was the opposite one — the text + * they raised was the raw tail, unscrubbed by the rules above, and carried no + * HTTP status for the run loop to classify on. `scrubAdapterFailure` below is + * the shared reader they now both go through, so the two rules cannot drift + * apart between three adapters. + */ + +/** The kept detail is truncated, ellipsis included, to this many characters. */ +export const MAX_REFUSAL_DETAIL = 256; + +/** + * The status an adapter prefixes to its error text: `"429: {…}"` -> `429`. + * Null for anything that is not a 4xx/5xx, so a message that merely starts with + * a number ("4040 tokens over budget") is not read as a status. + * + * The separator pi uses is NOT fixed — measured 2026-09-22, the same pi 0.84.1 + * emits `"429: {"message":…}"` for a JSON body and `"502 …"` for an + * HTML one. Both forms are matched here; a parser that required the colon would + * have kept `502 …` as the whole "body" and then dropped it as non-JSON, + * reporting the status alone with no sign that the body was never read. + */ +const STATUS_PREFIX = /^\s*(\d{3})(?!\d)\s*:?\s*/; + +export const statusFromError = (value) => { + const match = STATUS_PREFIX.exec(String(value ?? '')); + if (!match) return null; + const status = Number(match[1]); + return status >= 400 && status <= 599 ? status : null; +}; + +/** + * The body the adapter carried after its status prefix, or the whole text when + * there is no status to strip. `"429: {"message":"a: b"}"` -> `{"message":"a: b"}`; + * `"502 "` -> ``. + */ +export const bodyFromError = (value) => { + const text = String(value ?? ''); + if (statusFromError(text) === null) return text.trim(); + return text.replace(STATUS_PREFIX, '').trim(); +}; + +/** The shortest value the refusal-text reader treats as a credential. */ +export const MIN_CREDENTIAL_LENGTH = 8; + +/** + * Replace credential values this spawn was handed with a marker. + * + * This is the exact-match half, for text that is kept for its whole shape rather + * than filtered to one field — an adapter's stderr tail, say, whose diagnostic + * value is in the lines around the secret. Values shorter than eight characters + * are ignored: redacting a one-character "credential" would corrupt ordinary + * prose, and no issued credential is that short. + * + * The floor belongs to this reader only. The trace store scrubs every value + * registered with it, whatever its length, and the valet refuses to grant a + * credential shorter than this (credential_denied), so no granted value is + * ever below the floor. + */ +export const redactKnownCredentials = (text, credentials = []) => { + let out = String(text ?? ''); + for (const value of credentials) { + if (typeof value !== 'string' || value.length < MIN_CREDENTIAL_LENGTH) continue; + out = out.split(value).join('[redacted]'); + } + return out; +}; + +/** + * The keep-list. Given the raw body an adapter captured, keep at most one named + * field of it and only when that field is a string: + * + * `error.message` first (the OpenAI-shaped error), then a top-level `message` + * (which is the shape LiteLLM's budget refusal actually uses — the motivating + * case; a keep-list that kept only `error.message` would name the status and + * still hide the budget line that started this row). + * + * Everything else — HTML, a non-JSON body, a JSON body with neither field, a + * body echoing a known credential — contributes nothing. Returns null rather + * than a partial guess. + */ +export const keptRefusalDetail = (raw, { credentials = [] } = {}) => { + let parsed; + try { + parsed = JSON.parse(String(raw ?? '')); + } catch { + return null; + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null; + + const candidate = parsed?.error?.message ?? parsed?.message; + if (typeof candidate !== 'string') return null; + + const text = candidate.replace(/\s+/g, ' ').trim(); + if (!text) return null; + + const redacted = redactKnownCredentials(text, credentials); + // `[redacted]` appearing is the signal that the body echoed something this + // spawn was handed. The status is still worth keeping; the text is not. + if (redacted !== text) return null; + + return text.length > MAX_REFUSAL_DETAIL + ? `${text.slice(0, MAX_REFUSAL_DETAIL - 1)}…` + : text; +}; + +/** + * Read a pi JSON stream and return the refusal it ended on, or null. + * + * `auto_retry_end` is the terminal word — it names the status and the body after + * the ladder has finished — so it wins over the per-attempt `message_end` + * events that precede it. A run that refused on its first attempt and never + * retried still reports through `message_end`. + */ +export const readUpstreamRefusal = (stdout, { credentials = [] } = {}) => { + let status = null; + let raw = null; + + for (const line of String(stdout ?? '').split('\n')) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + + if (event?.type === 'auto_retry_end' && event.success === false) { + const candidate = event.finalError ?? event.errorMessage; + const attemptStatus = statusFromError(candidate); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(candidate); + } + continue; + } + + const message = event?.message; + if ( + event?.type === 'message_end' + && message?.role === 'assistant' + && message.stopReason === 'error' + && status === null + ) { + const attemptStatus = statusFromError(message.errorMessage); + if (attemptStatus) { + status = attemptStatus; + raw = bodyFromError(message.errorMessage); + } + } + } + + if (status === null) return null; + return { status, detail: keptRefusalDetail(raw, { credentials }) }; +}; + +/** + * Is this text STRUCTURED upstream data — a JSON object or an array? The gate for + * the whole-tail fallback below, deliberately NOT `kept === null`, which is true + * for five different reasons (Vera 71350). + * + * An array counts, and that is a correction (Vera 71360, measured): `[{...}]` is + * structured data with fields this code does not know, which is exactly what rule + * 1 exists for, and treating it as prose shipped the whole body under exact-match + * alone. A JSON *string* is the opposite case and does NOT count — `"Too many + * requests"` is a whole-shape text with no fields to reduce, so it keeps its + * wording like any other prose. + */ +const isStructuredBody = (raw) => { + try { + const parsed = JSON.parse(String(raw ?? '')); + return parsed !== null && typeof parsed === 'object'; + } catch { + return false; + } +}; + +/** + * A 3-digit number is read as a status only when the tail NAMES it as one: + * `status 429`, `HTTP 429`, `code: "429"` — or when it opens the tail or a ` | ` + * segment followed by a colon, which is the shape claude's `stderr | stdout` + * join produces for a body that leads with its status. + * + * A bare `\b(\d{3})\b` scan is not enough, and the false positives are not + * hypothetical: `500 tokens`, `"max_tokens": 500` and `retry in 500 seconds` + * all read as 500, and `classifySpawnFailure` tests the status FIELD before the + * message — so a count misread as a status stands the fleet down on a refusal + * that never happened, which is worse than no status at all (Vera 71352). + */ +const STATUS_NAMED = /\b(?:status(?:_?code)?|code|http)\b\W{0,4}(\d{3})(?!\d)/gi; +// `HTTP/1.1 429 Too Many Requests` is the most canonical refusal shape there is, +// and the `\W{0,4}` gap above handles `HTTP 429` but cannot reach a VERSIONED +// line: the `1` in `/1.1` is a +// WORD character, so `HTTP` and `429` are not within four non-word characters of +// each other. It read as `status: null` — an unclassified runtime failure on the +// single most recognisable shape, which is the misclassification this row exists +// to remove (Vera 71359). Its own pattern, not a looser gap: widening the gap +// would start reading `/v1/models 500` as a status. +const STATUS_LINE = /\bHTTP\/\d(?:\.\d)?\s+(\d{3})(?!\d)/gi; +const STATUS_COLON = /(?:^|\|\s*)(\d{3})(?!\d)\s*:/g; + +/** The first 4xx/5xx one of those shapes names, or null. */ +const scanStatuses = (text, pattern) => { + for (const match of String(text ?? '').matchAll(pattern)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) return value; + } + return null; +}; + +/** + * Find the status AND the body boundary, because they are the same question. + * + * Anchoring the status to a ` | ` segment start is not enough on its own: the + * body has to be taken from just past that status too, or a mid-tail read + * isolates the status and then hands the WHOLE joined tail to the body rules, + * which no longer look like a JSON body and so fall through to "keep it all". + * That is the difference between dropping a credential-echoing body and shipping + * the part of it that exact-match cannot see. + * + * `STATUS_COLON` is the only shape that moves the boundary, because it is the + * same shape as the anchored prefix (`429: {body}`), just found at a segment + * boundary — claude's `stderr | stdout` join. A status that appears inside a + * body (`{"error":{"code":"429"}}`) or as prose (`HTTP 503`) names the status + * (`HTTP/1.1 429`) but marks no boundary, so the whole text stays the body. + */ +const readStatusAndBody = (text) => { + const raw = String(text ?? ''); + const prefixed = statusFromError(raw); + if (prefixed !== null) return { status: prefixed, body: bodyFromError(raw) }; + // Only the colon shape moves the boundary — see the note above. A status LINE + // names the status but its reason phrase is prose, so the body stays the whole + // text and the exact-match rule protects it. + for (const match of raw.matchAll(STATUS_COLON)) { + const value = Number(match[1]); + if (value >= 400 && value <= 599) { + return { status: value, body: raw.slice(match.index + match[0].length).replace(/^\s*:?\s*/, '') }; + } + } + const named = scanStatuses(raw, STATUS_NAMED); + return { status: named ?? scanStatuses(raw, STATUS_LINE), body: raw }; +}; + +/** + * The shared reader for an adapter's own failure tail (TASK-103). + * + * The two rules above are about a body an adapter captured, and they apply here + * too — but the tail is a different shape from pi's refusal, and the difference + * decides how each rule lands: + * + * - The tail is often NOT structured. Claude's `-p` mode writes its terminal + * condition as prose (`Claude usage limit reached. Your limit will reset at + * 11:40pm.`), and that line is the whole diagnostic value of reporting + * stdout at all — 361 consecutive failures on 2026-08-03 carried no reason + * because of it. So the keep-list applies only when the tail parses as a + * body; when it does not, the text is KEPT as a whole shape and the + * exact-match rule is what protects it (`redactKnownCredentials`, which + * substitutes rather than drops, precisely because a whole-shape text has no + * second field to fall back to). + * + * - A body that IS structured (a JSON object or array) is reduced to one named + * field, and a body whose field echoed a value this spawn was handed is + * DROPPED WHOLE — the same trade rule 2 makes everywhere. The fallback to the + * whole tail is gated on the SHAPE (is it structured at all), never on + * "the keep-list returned null": that null has + * four other causes, and one of them is the credential echo this line exists + * to drop. Gating on `kept === null` would emit the whole body for that case, + * which is precisely what rule 2 forbids — and exact-match cannot see a + * transformed shape (limit (a) in the header), so a key that arrived + * base64'd or percent-encoded would ship (Vera 71350/71351). + * + * Returns `{ status, detail }`: `status` is the 4xx/5xx the tail named, or null. + * The caller decides where the status goes; `classifySpawnFailure` reads it off + * `error.status` or out of the message text, so both a leading `429: …` prefix + * and an attached field classify the same way. `detail` is `''` when nothing of + * the body may be shown — the status is then the whole diagnostic. + */ +export const scrubAdapterFailure = (text, { credentials = [], limit = MAX_REFUSAL_DETAIL } = {}) => { + const raw = String(text ?? '').trim(); + const { status, body } = readStatusAndBody(raw); + const kept = keptRefusalDetail(body, { credentials }); + const detail = kept ?? (isStructuredBody(body) ? '' : redactKnownCredentials(body, credentials)); + return { status, detail: detail.slice(0, limit) }; +}; + +/** + * The Error an adapter raises, built from its own failure tail. + * + * `status` is attached as a field, not only written into the text: the run + * loop's classifier tests `error.status === 429` before it looks at the + * message, and a refusal that is only readable by regex is one wording away + * from the misclassification that cost an hour on 2026-08-18 (see + * spawn-retry.js). The message still names the status so a human log says the + * same thing as the classifier's decision. + */ +export const adapterFailure = (label, text, { credentials = [], exitCode = null, limit } = {}) => { + const { status, detail } = scrubAdapterFailure(text, { credentials, limit }); + const code = exitCode === null ? '' : ` exited with code ${exitCode}`; + const named = status === null ? '' : ` (upstream ${status})`; + const error = new Error(`${label}${code}${named}${detail ? `: ${detail}` : ''}`); + if (status !== null) error.status = status; + return error; +}; + +/** + * The seat-log clause. The status is always present; the detail only when the + * keep-list kept one. `grep 'upstream refused'` finds every refusal, which is + * the property the old `no wrapper-post (empty output)` line did not have. + */ +export const describeUpstreamRefusal = (refusal) => { + if (!refusal || typeof refusal.status !== 'number') return null; + return refusal.detail + ? `upstream refused ${refusal.status}: ${refusal.detail}` + : `upstream refused ${refusal.status}`; +}; + +/** + * The credential values this spawn was handed, for the exact-match check in + * `redactKnownCredentials`. `envNames` are the adapter's own provider keys; the + * caller supplies them because only the adapter knows which key its route uses. + */ +export const spawnCredentials = (ctx = {}, envNames = []) => { + const env = ctx.env || process.env || {}; + return [ + ctx.runtimeToken, + ...envNames.map((name) => env[name]), + ].filter((value) => typeof value === 'string' && value.length >= MIN_CREDENTIAL_LENGTH); +}; diff --git a/docs/database/schema-migrations.md b/docs/database/schema-migrations.md index bb6a82f3fb..f2decedb7d 100644 --- a/docs/database/schema-migrations.md +++ b/docs/database/schema-migrations.md @@ -27,6 +27,8 @@ its rows do not determine which SQL files the runner executes. ## Adding a migration 1. Pick the next unused `NNNN_name.sql`. Coordinate numbers across open lanes. + Add its name to `SHIPPED` in `backend/__tests__/service/pgMigrations.test.js`; + two lanes taking the same number then conflict on that line. 2. Write SQL that upgrades a populated previous schema. A column inside `CREATE TABLE IF NOT EXISTS` alone never upgrades an existing table. Keep rolling old/new application versions compatible with additive changes. diff --git a/docs/plans/task-513-server-record.md b/docs/plans/task-513-server-record.md new file mode 100644 index 0000000000..b3f1fdb249 --- /dev/null +++ b/docs/plans/task-513-server-record.md @@ -0,0 +1,164 @@ +# TASK-513: server record boundary + +Status: server boundary ruled by Lily on 2026-10-25 at 15:45:17Z, decision +`6ade1e1d22a5cac4a3985799` (pod message 86051): **Task and pod chains**. +The CTO's 10-25 launch plan (task note, 2026-10-26 04:09Z) settles the two +items this document left open: record-v1 and its writer are on main +(`packages/seats`, #2739), and server lines carry no `argsDigest` at launch. + +Slice 1 (TASK-559) implements the store, the board's lines and the read +endpoint. Every writer and the endpoint are off unless the pod is listed in +`RECORD_V1_PODS`; nothing is public until a non-author walks it (slice 4). + +| slice | row | scope | +|---|---|---| +| 1 | TASK-559 | migration 0002, chain store, vendored writer, board lines, `GET /api/pods/:podId/record/tasks/:taskId` | +| 2 | TASK-560 | grant lines and the B1–B5 connector mappings | +| 3 | TASK-561 | `commonly_*` lines, a client-declared tool name checked against the route | +| 4 | TASK-562 | v2 view, browser checks, non-author walk | + +### Decisions taken in slice 1 + +- **One writer implementation, two copies.** The backend image builds from + `backend/` alone and is CommonJS; the writer is ESM in `packages/seats`. + `backend/vendor/seats/src` holds byte-identical copies of `format.js`, + `otel.js` and their imports, and a unit test fails when a copy differs or a + copied file imports one that was not copied. `npm run sync:seats-record` + refreshes them. The alternative, a second Docker build context, touches + every place the backend image is built (deploy-dev, deploy-staging, the kind + smoke test, both compose files). +- **runId per chain** is `srv-task-<32 hex>` or `srv-pod-<32 hex>`, a SHA-256 + over the pod and task (ADR-033 §8 q3, option a). The chain row stores it. +- **The head is stored beside the lines** (`record_chains.head_seq`, + `head_hash`) and served as `X-Record-Head` / `X-Record-Entries`, the spec's + outside record of the last line (§4). +- **Readers.** `canViewPod` decides existence (404 otherwise, so the route + confirms nothing). Lines then go to actual pod members only, agents + included. They carry approver and credential-owner ids verbatim, and a + hash-chained line cannot be redacted without breaking verification; those + are the ids `runTraces` withholds from admins and agent-DM observers, so + those readers get a 403. +- **Board lines** mirror committed transitions: a fresh claim, a takeover, a + resumed blocked task, or a holder's note that restores a lapsed claim writes + `task.claimed`; a renewal by the holder writes + nothing; blocking a claim or a lapsed lease writes `task.released`; every + path to done writes `task.done` with the board as actor. Reopening and + unblocking have no record-v1 kind and write nothing. `leaseUntil` comes only + from a write that set a lease (a claim, a restoring note). A PATCH to + `claimed` sets no lease, so its `task.claimed` line carries empty data. Each line is keyed by the row's + persisted `updatedAt`, which the revision CAS advances on every write. +- **Emission** runs after the source write commits, queued per chain in each + process so one process appends in commit order, and never changes the + source write's result. + +## Measured inputs (2026-10-25) + +- TASK-504 landed in #2684, merge `ab5132313ad399d8fb990d69bc39db0ccf99e055`. +- record-v1 is `packages/seats/docs/record-v1.md` on + `feat/jellymoon-record` at `fb686d4fdd3143d5fa5f4fbeb43f46c38c3cbf90`. + Its reference writer, verifier, OTLP exporter and digest vectors share that + commit. The package is not on the implementation base, `2b0ea6689`. +- ADR-033 at `e2c89e2f0a1032f210d581545054ed7e7d728ef4` remains Proposed. + Section 8 explicitly leaves the server run, taskless calls, storage and CAP + capture breadth open. These are decisions, not fields supplied by a caller. +- `RoomGrant` and `ToolCall` on this base have no task binding. A seat grant's + target id is a seat, so it cannot supply a pod id. `commonly_*` requests have + no shared task or run context in their HTTP client. +- `runTraces.ts` already provides a human-readable run view. It permits + `canViewPod` readers but withholds grant and credential-owner identities from + readers who are not actual origin members. A new record reader must state + its treatment of those identities explicitly. + +## Ruled boundary + +Use a server-owned PG chain per `(podId, taskId)`, with the backend as the sole +logical writer and transaction serialization across replicas. Give taskless +calls a separate pod chain, omitting `taskId`; do not guess the task from the +agent's most recent claim. A caller's optional task context must be validated +against server state before any line receives that task id. The exact grant +binding contract is shared with TASK-288. + +Capture metadata for every authenticated `commonly_*` call, reads included, +plus connector calls. Store the exact allowlisted tool name, actor, server +timestamp and outcome. Omit `argsDigest` from every server record line and +JSONL/OTLP export under the interim launch rule below. Do not store arguments, +response bodies, credentials, arbitrary headers or exception prose. A request +must not select another agent's identity, a foreign pod, or an unrelated task. +Requests without an attributable pod need a separately ruled boundary; never +invent a pod from a seat id or a grant id. + +Persist canonical record-v1 bytes at emit time, after the source write commits. +A per-chain lock and atomic head update preserve consecutive sequence numbers +and `prev` across replicas. Repeated delivery of one source event is idempotent. +A record failure leaves the original authorization/effect unchanged and emits +a bounded operational error. Readers return an error for unavailable storage; +they never present an outage as an empty complete record. + +Reuse the reference format/verifier/exporter from the pinned shared package. +Do not introduce a second serializer or copy an untracked implementation. +Coordinate its landing or a shared module extraction with its owner. + +Emit grants, approval requests and decisions, task transitions and tool events +with the ADR's B1–B5 mappings. Derive actors from authenticated identities and +committed source rows. Keep credential-owner approval unchanged and keep +`decision-owner` reserved. Do not add approval text until the known-credential +redaction path has its required end-to-end proof. + +Expose task records through the task's existing authorization boundary, with +current `canViewPod` checks before loading record bytes. Explicitly test and +document the treatment of admin/observer readers, public pods and forwarded +approval cards. Follow origin pod links, not the card's display pod. Reuse the +existing run-trace presentation where it fits; add one task or decision entry +point and JSONL/OTLP export, with English and Chinese copy. + +## Ruling and remaining integration dependencies + +Lily chose task and pod chains over real agent-run chains or a task-only first +slice. That settles sequencing ownership, taskless capture and breadth: one +backend-sequenced PG chain per task, separate pod chains for taskless calls, +and every pod-attributable commonly call, reads included. Explicit validated +context is required; a recent task claim is never a task-binding source. + +Implementation reuses the shared record-v1 writer/verifier/exporter, now on +main via #2739 and vendored byte-identically as described above. ADR-033 (#2691) +is marked Accepted but not merged. Its approval and stored parked-call binding +rules (D5) gate slice 2's approval lines, not slice 1. + +## Interim digest rule (2026-10-25) + +Rivet's handoff in pod message 87089 pins proposed ADR-033 §8 question 9 at +`d2dbd42bae7819b76c70e5972ddfe90f4981e2a4`. TASK-513 adopts its interim +launch rule: omit `argsDigest` from **all** server record lines and exports, +including B1–B5. Do not copy broker-internal `tool_calls.args_digest` into the +record. That internal value remains unchanged for approval integrity. + +CTO owns the later privacy decision between a separately specified keyed field +and omission for low-entropy tools. Do not add a keyed value to v1's `sha256:` +field: its canonical unkeyed meaning is fixed. Digest emission remains disabled +until the decision and conformance/privacy tests are accepted. An absent digest +never means a call had no arguments. The ADR remains Proposed; this handoff does +not ratify the rest of it or enable a writer. + +## Required evidence before enablement + +1. Shared writer/verifier round-trip and deterministic OTLP bytes; editing a + line breaks verification. Assert `argsDigest` is absent from persisted server + lines, JSONL and OTLP for every emitted kind, including B1–B5, even when the + source broker row has `args_digest`. A future digest field requires the + accepted privacy decision and all shared conformance vectors first. +2. Real PostgreSQL concurrent appends, idempotent replay, rollback on failed + head update and upgrade from the populated prior schema. +3. Cross-pod/task denial with no content read, public nonmember denial, + current membership removal, forwarded card origin checks and both auth + shapes. Test admin/observer identity redaction explicitly. +4. A real committed source write per grant, attenuation, revocation, expiry, + parked/approved/declined/timed-out call, task claim/release/completion, and + both connector and commonly tool surfaces. No duplicate terminal decision. +5. A taskless call remains taskless. Two concurrent tasks for one agent never + misattribute calls. Invalid context cannot widen access or forge an actor. +6. No resolved credential reaches record bytes; no request/response body or + vendor error text appears. An audit-store failure does not change the + source operation and never yields a successful empty export. +7. Real-browser checks at 390 and 1440, then a non-author walk of one real task + from claim through result, including JSONL and a stock OTLP viewer. Independent + Tier 2 code/security gates precede enablement. The author does not self-gate.