diff --git a/README.md b/README.md index 99bb34b..bd4eaa9 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,32 @@ categories, not the blend. Files are written per run, so a compare that dies midway still leaves the completed runs' records. Cache-served baseline arms ran in an earlier invocation and write nothing. +### Per-turn transcripts + +The raw result is still an aggregate: one `usage` total for the whole run. It +can tell you a prompt's uncached input stayed small; it cannot tell you how +context grew between turn 2 and turn 3, or which turn paid for the cache +write. `--transcript-out ` (on `compare` and `measure`, claude-p only) +runs claude with `--output-format stream-json --verbose` and writes the full +event stream as `__.stream.jsonl`, one JSON event per line: + +```bash +promptdiff compare --scenario ./scenario.json --transcript-out ./transcripts + +# per-turn input growth, cache reads, and cache writes for one run +jq -r 'select(.type == "assistant") | .message.usage + | [.input_tokens, .cache_read_input_tokens, .cache_creation_input_tokens] | @tsv' \ + transcripts/injected-context_proposed_1.stream.jsonl +``` + +Lines are written as they arrive, so a run killed by the timeout or a budget +abort keeps everything it printed up to the kill — for a cost investigation a +truncated stream is the evidence. The runner still passes +`--no-session-persistence`: capture replaces the transcript without writing +anything to `~/.claude/projects`. Stream files are large (every tool result +and every message block), so the flag is off by default and best pointed at a +throwaway directory. Cache-served baseline arms write nothing here either. + ### Run history `--report ndjson --report-out ./runs.ndjson` appends one record per scenario diff --git a/SPEC.md b/SPEC.md index 5243769..b74d0e5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -194,7 +194,11 @@ state. `--raw-out ` (compare and measure) additionally writes each run's full runner result JSON — token usage categories, modelUsage, subtype — one file per completed run, for token-level analysis that the digested summaries cannot support; files land as runs finish, so a partial invocation still -leaves its completed records. +leaves its completed records. `--transcript-out ` (compare and +measure, streamEvents runners only) additionally captures the runner's +per-event stream as NDJSON, one `__.stream.jsonl` per run +written event by event — per-turn usage and tool calls, which the aggregate +result object cannot carry. A run killed mid-stream keeps its partial file. `compare` scenarios assert nothing: they exist to report both arms' pass rates and the delta, for comparisons (typically model-vs-model) where neither diff --git a/src/cli.ts b/src/cli.ts index 25522c4..b1f5966 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -70,6 +70,7 @@ const compareSpecs: FlagSpecs = { "report-out": { arity: "one" }, receipts: { arity: "one" }, "raw-out": { arity: "one" }, + "transcript-out": { arity: "one" }, cache: { arity: "none" }, "cache-dir": { arity: "one" }, }; @@ -241,6 +242,7 @@ const measureSpecs: FlagSpecs = { "keep-sandbox": { arity: "none" }, receipts: { arity: "one" }, "raw-out": { arity: "one" }, + "transcript-out": { arity: "one" }, }; async function cmdMeasure(argv: string[]): Promise { @@ -272,12 +274,14 @@ async function cmdMeasure(argv: string[]): Promise { }; const rawOutDir = args.one("raw-out"); + const transcriptOutDir = args.one("transcript-out"); const config = loadCompareConfig(scenario, overrides, { singleArm: true }); const summary = await runMeasure({ config, runner: armRunner(config.arms.baseline, config), onProgress: (message) => console.error(`[promptdiff] ${message}`), rawOut: rawOutDir === undefined ? undefined : { dir: rawOutDir }, + transcriptOut: transcriptOutDir === undefined ? undefined : { dir: transcriptOutDir }, }); const receiptsDir = args.one("receipts"); @@ -369,6 +373,7 @@ async function cmdCompare(argv: string[]): Promise { }; const rawOutDir = args.one("raw-out"); + const transcriptOutDir = args.one("transcript-out"); const config = loadCompareConfig(scenario, overrides); const summary = await runCompare({ config, @@ -379,6 +384,7 @@ async function cmdCompare(argv: string[]): Promise { onProgress: (message) => console.error(`[promptdiff] ${message}`), cache, rawOut: rawOutDir === undefined ? undefined : { dir: rawOutDir }, + transcriptOut: transcriptOutDir === undefined ? undefined : { dir: transcriptOutDir }, }); // History is appended before the exit code is decided — failed comparisons @@ -545,7 +551,8 @@ function compareUsage(): string { " --mode --tools ", " --timeout-ms --max-budget-usd --max-turns ", " --report ndjson --report-out ", - " --raw-out --cache [--cache-dir ]", + " --raw-out --transcript-out ", + " --cache [--cache-dir ]", "", "turn cap:", " --max-turns (or scenario \"maxTurns\", per-case override allowed) caps", @@ -558,6 +565,14 @@ function compareUsage(): string { " run — the token-level record that summaries and reports digest away.", " Cache-served baseline arms ran earlier and write nothing here.", "", + "transcripts:", + " --transcript-out runs claude with --output-format stream-json and", + " writes the full per-event NDJSON as __.stream.jsonl,", + " one line per event as it arrives — per-turn usage (cache reads, cache", + " creation, context growth) and tool calls, which the aggregate result", + " cannot show. Files are large; a run killed by the timeout keeps the", + " partial stream. claude-p only; cache-served arms write nothing.", + "", "caching:", " --cache reuses recorded baseline-arm results (default dir .promptdiff/cache)", " when nothing that could change the outcome changed: rendered prompts, skill", @@ -664,7 +679,7 @@ function measureUsage(): string { " --mode --tools ", " --sandbox --seed --keep-sandbox", " --timeout-ms --max-budget-usd --max-turns ", - " --receipts --raw-out ", + " --receipts --raw-out --transcript-out ", "", "--receipts writes one .receipt.json per scenario with", "per-file prompt hashes and the measured rates (verdict \"measured\").", @@ -672,6 +687,8 @@ function measureUsage(): string { "--max-turns caps agentic turns per run (claude-p); a capped run is", "scored as a failure without grading. --raw-out writes each run's", "full runner result JSON (token usage included) as _measure_.json.", + "--transcript-out additionally captures the per-event stream-json", + "NDJSON as _measure_.stream.jsonl (claude-p only, large).", "", "Exit code is 0 whenever the runs complete — a measurement has no pass/fail.", ].join("\n"); diff --git a/src/engine/compare.ts b/src/engine/compare.ts index 4c8ac3e..bec1851 100644 --- a/src/engine/compare.ts +++ b/src/engine/compare.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { closeSync, mkdirSync, openSync, writeFileSync, writeSync } from "node:fs"; import { join } from "node:path"; import { assembleSystemPrompt } from "../prompt"; import type { Runner, RunnerRunOptions, RunResult } from "../types"; @@ -68,15 +68,26 @@ export interface CompareRunOptions { * nothing here. */ rawOut?: { dir: string }; + /** + * Opt-in per-run transcript capture: the runner's full event stream is + * written to this directory as NDJSON while the run is in flight. Needs a + * streamEvents runner. Cache-served baseline arms write nothing here. + */ + transcriptOut?: { dir: string }; } export async function runCompare(options: CompareRunOptions): Promise { - const { config, runners, onProgress, cache, rawOut } = options; + const { config, runners, onProgress, cache, rawOut, transcriptOut } = options; // Validate each arm against its own runner — a mixed comparison fails on the // violating arm before either arm's paid runs. - validateRunnerSupport(config, runners.baseline); - validateRunnerSupport(config, runners.proposed); + const transcripts = transcriptOut !== undefined; + validateRunnerSupport(config, runners.baseline, { transcripts }); + validateRunnerSupport(config, runners.proposed, { transcripts }); assertJudgeGradersCalibrated(config); + // An unwritable (or file-blocked) transcript dir must fail here, once, before + // scenario 1 prepares a sandbox — not inside runArm, where prepareSandbox has + // already run and would otherwise leak an unclosed sandbox directory. + if (transcriptOut !== undefined) mkdirSync(transcriptOut.dir, { recursive: true }); // Install delivery keeps skill text out of the system prompt entirely — the // arms differ only by which skill directory lands in the sandbox registry. const inline = config.delivery !== "install"; @@ -89,8 +100,8 @@ export async function runCompare(options: CompareRunOptions): Promise void; /** Opt-in per-run raw persistence; see CompareRunOptions.rawOut. */ rawOut?: { dir: string }; + /** Opt-in per-run transcript capture; see CompareRunOptions.transcriptOut. */ + transcriptOut?: { dir: string }; } /** @@ -297,9 +322,11 @@ export interface MeasureRunOptions { * nonsense verdicts from sampling noise. */ export async function runMeasure(options: MeasureRunOptions): Promise { - const { config, runner, onProgress, rawOut } = options; - validateRunnerSupport(config, runner); + const { config, runner, onProgress, rawOut, transcriptOut } = options; + validateRunnerSupport(config, runner, { transcripts: transcriptOut !== undefined }); assertJudgeGradersCalibrated(config); + // Same fail-before-any-paid-run guarantee as runCompare: see the comment there. + if (transcriptOut !== undefined) mkdirSync(transcriptOut.dir, { recursive: true }); const inline = config.delivery !== "install"; const basePrompt = assembleSystemPrompt(config.agent, inline ? config.baselineSkills : []); // Same fail-before-any-paid-run guarantee as compare: render everything first. @@ -315,7 +342,7 @@ export async function runMeasure(options: MeasureRunOptions): Promise void, rawOut?: { dir: string }, + transcriptOut?: { dir: string }, ): Promise { if (cache === undefined) { - return runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress, "baseline", rawOut); + return runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress, "baseline", rawOut, transcriptOut); } const key = buildCacheKey({ systemPrompt, @@ -421,9 +449,12 @@ async function runBaselineArm( if (rawOut !== undefined) { onProgress?.(" baseline: cache hit — no raw results to write"); } + if (transcriptOut !== undefined) { + onProgress?.(" baseline: cache hit — no transcript to write"); + } return { ...hit, cached: true }; } - const summary = await runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress, "baseline", rawOut); + const summary = await runArm(config, evalCase, "baseline", systemPrompt, runner, onProgress, "baseline", rawOut, transcriptOut); storeCachedArm(cache.dir, key, summary); return summary; } @@ -437,6 +468,7 @@ async function runArm( onProgress?: (message: string) => void, label: string = arm, rawOut?: { dir: string }, + transcriptOut?: { dir: string }, ): Promise { const runs = evalCase.runs ?? config.runs; const summaries: ArmRunSummary[] = []; @@ -450,7 +482,15 @@ async function runArm( keep: config.keepSandbox, }); + let transcript: TranscriptSink | undefined; + try { + // Opened inside the try, after the sandbox exists, so the finally below + // covers both: a failed open (dir gone unwritable mid-run) and a failed + // run leave the sandbox cleaned up. Opened before the run itself so a + // timeout kill still leaves the lines the run did print — a partial + // stream is the evidence, not garbage to discard. + transcript = transcriptOut === undefined ? undefined : openTranscript(transcriptOut.dir, evalCase.name, label, index + 1); onProgress?.(` ${label} run ${index + 1}/${runs}`); if (config.delivery === "install") { const { installed, warnings } = installSkills(armSkills, sandbox.dir); @@ -461,7 +501,9 @@ async function runArm( } } } - const run = await runner.run(buildRunnerOptions(config, evalCase, arm, systemPrompt, sandbox.dir)); + const run = await runner.run( + buildRunnerOptions(config, evalCase, arm, systemPrompt, sandbox.dir, transcript?.write), + ); if (rawOut !== undefined) { writeRawResult(rawOut.dir, evalCase.name, label, index + 1, run.raw); } @@ -478,6 +520,7 @@ async function runArm( }); summaries.push(toArmRunSummary(index + 1, run, grade, sandbox.dir)); } finally { + transcript?.close(); sandbox.cleanup(); } } @@ -500,6 +543,7 @@ function buildRunnerOptions( arm: "baseline" | "proposed", systemPrompt: string, cwd: string, + onStreamEvent?: (line: string) => void, ): RunnerRunOptions { return { systemPrompt, @@ -514,6 +558,7 @@ function buildRunnerOptions( timeoutMs: config.timeoutMs, maxBudgetUsd: config.maxBudgetUsd, maxTurns: evalCase.maxTurns ?? config.maxTurns, + ...(onStreamEvent === undefined ? {} : { onStreamEvent }), }; } @@ -525,8 +570,55 @@ function buildRunnerOptions( */ function writeRawResult(dir: string, caseName: string, label: string, runNumber: number, raw: unknown): void { mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, `${runFileStem(caseName, label, runNumber)}.json`), JSON.stringify(raw, null, 2) + "\n", "utf8"); +} + +interface TranscriptSink { + /** Appends one NDJSON event line; a no-op once the file is closed. */ + write: (line: string) => void; + close: () => void; +} + +/** + * One NDJSON file per run, written event by event. The runner emits lines and + * never learns what a scenario is; naming and the keep-the-partial policy live + * here, next to raw persistence, so both answer to one convention. + */ +function openTranscript(dir: string, caseName: string, label: string, runNumber: number): TranscriptSink { + mkdirSync(dir, { recursive: true }); + const fd = openSync(join(dir, `${runFileStem(caseName, label, runNumber)}.stream.jsonl`), "w"); + let open = true; + return { + write(line: string): void { + if (!open) return; + const payload = Buffer.from(line.endsWith("\n") ? line : `${line}\n`, "utf8"); + // writeSync may legally write fewer bytes than asked (a signal, a full + // disk) — a short write would truncate an NDJSON line and corrupt the + // evidence these files exist to preserve. Loop on byte offsets, not + // string indices: slicing the string by the returned count would + // misalign on any multi-byte UTF-8 character. + let written = 0; + while (written < payload.length) { + const n = writeSync(fd, payload, written, payload.length - written); + // A zero-byte write is legal and makes no progress — looping on it + // would hang the run inside the runner's event callback. Throwing + // bounds it: the caller's finally still closes the transcript and + // cleans the sandbox, and the lines already written stay on disk. + if (n <= 0) throw new Error(`transcript write stalled at ${written}/${payload.length} bytes`); + written += n; + } + }, + close(): void { + if (!open) return; + open = false; + closeSync(fd); + }, + }; +} + +function runFileStem(caseName: string, label: string, runNumber: number): string { const safeCase = caseName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "scenario"; - writeFileSync(join(dir, `${safeCase}_${label}_${runNumber}.json`), JSON.stringify(raw, null, 2) + "\n", "utf8"); + return `${safeCase}_${label}_${runNumber}`; } function effectiveTools(config: CompareConfig, evalCase: EvalCaseConfig): string { diff --git a/src/runner/claude-p.ts b/src/runner/claude-p.ts index a8f5ae6..f03bc24 100644 --- a/src/runner/claude-p.ts +++ b/src/runner/claude-p.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import type { RunResult, Runner, RunnerRunOptions } from "../types"; interface ClaudeJsonResult { + type?: unknown; result?: unknown; subtype?: unknown; total_cost_usd?: unknown; @@ -29,12 +30,19 @@ export function buildClaudeArgs(options: RunnerRunOptions & { systemPromptFile: ? ["--tools", options.tools] : ["--tools", options.tools, "--allowedTools", options.tools]; + // A transcript sink needs the per-event stream; the terminal event of that + // stream is the same result object --output-format json prints on its own. + // stream-json is only valid with --verbose under -p. + const outputArgs = + options.onStreamEvent === undefined + ? ["--output-format", "json"] + : ["--output-format", "stream-json", "--verbose"]; + const args = [ "-p", options.userPrompt, ...systemPromptArgs, - "--output-format", - "json", + ...outputArgs, "--model", options.model, ...toolArgs, @@ -59,7 +67,7 @@ export function buildClaudeArgs(options: RunnerRunOptions & { systemPromptFile: export class ClaudePrintRunner implements Runner { readonly name = "claude-p"; - readonly capabilities = { sandboxTools: true, skillRegistry: true, images: false }; + readonly capabilities = { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }; constructor(private readonly claudeBin = "claude") {} @@ -83,8 +91,9 @@ export class ClaudePrintRunner implements Runner { setTimeout(() => proc.kill("SIGKILL"), 2_000); }, options.timeoutMs); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), + const sink = options.onStreamEvent; + const [out, stderr, code] = await Promise.all([ + sink === undefined ? readWholeStdout(proc.stdout) : consumeStreamJson(proc.stdout, sink), new Response(proc.stderr).text(), proc.exited, ]); @@ -97,19 +106,45 @@ export class ClaudePrintRunner implements Runner { // A turn-cap stop is a measured outcome, not a crash: claude may exit // non-zero with the full result JSON on stdout. Return it so the engine // can score the run as a failure instead of aborting the comparison. - const capped = tryParseClaudeJson(stdout); + // Only a terminal event counts — a result-shaped event with more stream + // after it says nothing about how the run ended, and scoring it would + // turn a crash into a data point. + const terminal = out.resultIsTerminal ? out.result : undefined; + const capped = terminal ?? tryParseClaudeJson(out.text); if (capped?.subtype === "error_max_turns") { return normalizeClaudeResult(capped); } - throw new Error(describeClaudeFailure(code, stdout, stderr, options.maxBudgetUsd)); + // A non-terminal event still beats a bare exit code in an error message, + // and a message is not a score — so this one keeps using out.result. + throw new Error(describeClaudeFailure(code, out.text, stderr, options.maxBudgetUsd, out.result)); + } + + if (sink !== undefined) { + // A stream killed mid-flight (SIGTERM, a crashed CLI) ends without its + // terminal event, so "the last line is the result" is not safe. Neither + // is a result event that fired mid-stream and was then followed by more + // output, even on a clean exit. Either way there is no outcome to score, + // and the lines already written stay on disk as evidence. The two causes + // get separate messages: a run that hard-fails here is only worth failing + // loudly if whoever reads the message can tell which one happened. + if (out.result === undefined) { + throw new Error(`claude produced no terminal result event${tailForMessage(out.text)}`); + } + if (!out.resultIsTerminal) { + throw new Error( + `claude emitted a result event before the end of the stream — refusing to score a ` + + `non-terminal event${tailForMessage(out.text)}`, + ); + } + return normalizeClaudeResult(out.result); } let parsed: ClaudeJsonResult; try { - parsed = JSON.parse(stdout) as ClaudeJsonResult; + parsed = JSON.parse(out.text) as ClaudeJsonResult; } catch (error) { const reason = error instanceof Error ? error.message : String(error); - throw new Error(`claude returned invalid JSON: ${reason}\n${stdout.slice(0, 1_500)}`); + throw new Error(`claude returned invalid JSON: ${reason}\n${out.text.slice(0, 1_500)}`); } return normalizeClaudeResult(parsed); @@ -129,8 +164,10 @@ export function describeClaudeFailure( stdout: string, stderr: string, maxBudgetUsd: number, + /** Terminal result event, already decoded — stream-json stdout will not parse whole. */ + streamResult?: ClaudeJsonResult, ): string { - const parsed = tryParseClaudeJson(stdout); + const parsed = streamResult ?? tryParseClaudeJson(stdout); if (parsed && typeof parsed === "object") { if (parsed.subtype === "error_max_budget_usd") { @@ -146,6 +183,94 @@ export function describeClaudeFailure( return `claude exited ${code}: ${detail.slice(0, 1_500)}`; } +/** Bytes of stdout kept for diagnostics when the stream is too big to hold. */ +const STDOUT_TAIL_CHARS = 4_000; + +interface ClaudeStdout { + /** stdout text for error messages: all of it in json mode, a tail in stream mode. */ + text: string; + /** Terminal `result` event; only stream mode decodes one while reading. */ + result?: ClaudeJsonResult; + /** + * Whether `result` was the last line the stream produced, rather than a + * result-shaped event with more stream (or a truncated fragment) after it. + * Only a terminal event may be scored as a run's outcome. + */ + resultIsTerminal: boolean; +} + +async function readWholeStdout(stdout: ReadableStream): Promise { + // json mode decodes no event stream, so it has no terminal result event; + // its parse runs on the whole buffered text instead. + return { text: await new Response(stdout).text(), resultIsTerminal: false }; +} + +/** + * Reads NDJSON as it arrives, hands each line to the sink, and keeps only the + * terminal `result` event. Buffering the whole verbose stream to parse it + * afterwards is the memory cost transcript capture exists to avoid, and a + * killed run would lose every line it had already printed. + */ +async function consumeStreamJson( + stdout: ReadableStream, + onStreamEvent: (line: string) => void, +): Promise { + const decoder = new TextDecoder(); + let pending = ""; + let tail = ""; + let result: ClaudeJsonResult | undefined; + let resultIsTerminal = false; + + const handleLine = (line: string): void => { + if (line.trim().length === 0) return; + onStreamEvent(line); + tail = (tail + line + "\n").slice(-STDOUT_TAIL_CHARS); + const event = tryParseClaudeJson(line); + // The last result event wins when more than one appears. A subagent or + // compaction could in principle emit one mid-stream, so callers get told + // whether the stream ended on it: the success path trusts out.result only + // after a clean exit, and the failure path only when it was terminal. + // Any later line — another event, or a truncated trailing fragment — + // demotes it. + if (event?.type === "result") { + result = event; + resultIsTerminal = true; + } else { + resultIsTerminal = false; + } + }; + + for await (const chunk of stdout) { + // `pending` up to this length was already scanned for "\n" with none + // found (that's why the previous iteration's while loop exited) — only + // the newly appended bytes need to be searched. Once a line is sliced + // off below, the remaining buffer is unscanned from its own start 0. + // Accumulating into a string looks quadratic and measures linear: the + // engine ropes the `+=` instead of copying, and this offset keeps the + // scan off old bytes. A 16MB single line costs ~3ms, a 4MB one under 1ms, + // so a byte-buffer rewrite would add UTF-8 boundary handling here for no + // gain worth having. + const scanned = pending.length; + pending += decoder.decode(chunk, { stream: true }); + let newline = pending.indexOf("\n", scanned); + while (newline !== -1) { + handleLine(pending.slice(0, newline)); + pending = pending.slice(newline + 1); + newline = pending.indexOf("\n"); + } + } + pending += decoder.decode(); + // A kill mid-line leaves an unterminated fragment. It is not parseable, but + // for a token-economics investigation a truncated event is still evidence. + if (pending.length > 0) handleLine(pending); + + return { text: tail, result, resultIsTerminal }; +} + +function tailForMessage(text: string): string { + return text.trim().length === 0 ? "" : ` (last output: ${text.slice(-500)})`; +} + function tryParseClaudeJson(stdout: string): ClaudeJsonResult | undefined { try { return JSON.parse(stdout) as ClaudeJsonResult; diff --git a/src/runner/openai-compat.ts b/src/runner/openai-compat.ts index c69059b..514b39f 100644 --- a/src/runner/openai-compat.ts +++ b/src/runner/openai-compat.ts @@ -94,7 +94,7 @@ export function buildChatRequest(options: RunnerRunOptions): ChatRequest { */ export class OpenAiCompatRunner implements Runner { readonly name = "openai"; - readonly capabilities = { sandboxTools: false, skillRegistry: false, images: true }; + readonly capabilities = { sandboxTools: false, skillRegistry: false, images: true, streamEvents: false }; private readonly baseUrl: string; private readonly apiKey: string | undefined; diff --git a/src/types.ts b/src/types.ts index 298e880..facbb19 100644 --- a/src/types.ts +++ b/src/types.ts @@ -37,6 +37,11 @@ export interface RunnerCapabilities { * required for scenarios with `images`. */ images: boolean; + /** + * The runner can emit its provider's per-event stream to `onStreamEvent` — + * required for transcript capture. + */ + streamEvents: boolean; } export interface RunnerRunOptions { @@ -64,6 +69,13 @@ export interface RunnerRunOptions { maxBudgetUsd: number; /** Turn cap for agentic runners; completion runners are single-turn and ignore it. */ maxTurns?: number; + /** + * Per-event sink for transcript capture: the runner emits one NDJSON line + * per provider event and never learns where the lines land. Naming, the + * partial-write-on-failure policy, and closing the file are the engine's. + * Runners without a streaming mode leave it unused. + */ + onStreamEvent?: (line: string) => void; } export interface Runner { diff --git a/test/cache.test.ts b/test/cache.test.ts index 5416e9b..93f68f8 100644 --- a/test/cache.test.ts +++ b/test/cache.test.ts @@ -14,7 +14,7 @@ function makeCountingRunner(): CountingRunner { const counts = { baseline: 0, proposed: 0 }; return { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, counts, async run(options: RunnerRunOptions) { if (options.systemPrompt.includes("PROPOSED")) { @@ -191,3 +191,54 @@ test("compare rejects --cache-dir without --cache before doing any work", () => expect(result.exitCode).toBe(2); expect(result.stderr.toString()).toContain("--cache-dir requires --cache"); }); + +test("a cached baseline arm says it has no transcript to write instead of writing an empty one", async () => { + const fixture = makeFixture(); + try { + const cache = { dir: fixture.cacheDir }; + // The counting runner stands in for claude-p here: capture demands a + // streamEvents runner, which the engine checks before any run. + const streamingRunner = (): CountingRunner => { + const runner = makeCountingRunner(); + return { ...runner, capabilities: { ...runner.capabilities, streamEvents: true } }; + }; + const first = streamingRunner(); + const firstTranscriptOut = { dir: join(fixture.dir, "transcripts-1") }; + await runCompare({ + config: fixture.config, + runners: { baseline: first, proposed: first }, + cache, + transcriptOut: firstTranscriptOut, + }); + // Sanity check on the harness itself: a genuine (cache-miss) baseline run + // does open transcript files, so the second run's directory is a fair test. + expect(readdirSync(firstTranscriptOut.dir).some((name) => name.includes("_baseline_"))).toBe(true); + + // A fresh directory per run makes "no baseline file was created" directly + // assertable, rather than inferring it from mtimes on a shared directory + // that the proposed arm (never cached) also writes into on every run. + const progress: string[] = []; + const second = streamingRunner(); + const secondTranscriptOut = { dir: join(fixture.dir, "transcripts-2") }; + await runCompare({ + config: fixture.config, + runners: { baseline: second, proposed: second }, + cache, + transcriptOut: secondTranscriptOut, + onProgress: (message) => progress.push(message), + }); + + // The arm ran in an earlier invocation; capture cannot reach back for it. + expect(second.counts.baseline).toBe(0); + expect(progress).toContain(" baseline: cache hit — no transcript to write"); + // The actual invariant the test name promises: a cache-served baseline + // opens no transcript file at all, empty or otherwise. The proposed arm + // (not cached) still writes its own, which also proves transcript capture + // was live for this run and not just skipped wholesale. + const secondRunFiles = readdirSync(secondTranscriptOut.dir); + expect(secondRunFiles.some((name) => name.includes("_baseline_"))).toBe(false); + expect(secondRunFiles.some((name) => name.includes("_proposed_"))).toBe(true); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); diff --git a/test/compare.test.ts b/test/compare.test.ts index 1696192..e3c53dd 100644 --- a/test/compare.test.ts +++ b/test/compare.test.ts @@ -18,7 +18,7 @@ test("runCompare verifies target improvement and regression preservation", async const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { const isProposed = options.systemPrompt.includes("PROPOSED"); const isRegression = options.userPrompt.includes("regression"); @@ -93,7 +93,7 @@ test("runCompare routes each arm to its own runner and model, and compare kind n const makeRunner = (name: string, seen: string[], output: string): Runner => ({ name, - capabilities: { sandboxTools: false, skillRegistry: false, images: false }, + capabilities: { sandboxTools: false, skillRegistry: false, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { seen.push(options.model); return { output, costUsd: 0, turns: 1, durationMs: 10, models: [options.model], raw: {} }; @@ -258,7 +258,7 @@ test("a run that exhausted its turn cap fails without grading its partial output // Output would satisfy the grader — the cap must fail the run anyway. const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { expect(options.maxTurns).toBe(7); const capped = options.systemPrompt.includes("BASELINE"); @@ -289,7 +289,7 @@ test("rawOut persists each run's full runner result for both compare arms and fo const usage = { input_tokens: 100, cache_read_input_tokens: 900, output_tokens: 50 }; const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { const arm = options.systemPrompt.includes("BASELINE") ? "baseline" : "proposed"; return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: ["sonnet"], raw: { arm, usage } }; @@ -317,3 +317,189 @@ test("rawOut persists each run's full runner result for both compare arms and fo rmSync(fixture.dir, { recursive: true, force: true }); } }); + +test("transcriptOut captures each run's event stream, keeping the partial when a run throws", async () => { + const fixture = fixtureDir(); + try { + const streamingRunner = (fail: boolean): Runner => ({ + name: "mock-stream", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }, + async run(options: RunnerRunOptions) { + const arm = options.systemPrompt.includes("BASELINE") ? "baseline" : "proposed"; + options.onStreamEvent?.(JSON.stringify({ type: "system", arm })); + options.onStreamEvent?.(JSON.stringify({ type: "assistant", usage: { cache_read_input_tokens: 900 } })); + if (fail) throw new Error("killed mid-run"); + options.onStreamEvent?.(JSON.stringify({ type: "result", result: "ok" })); + return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: ["sonnet"], raw: {} }; + }, + }); + + const config = cappedConfig(fixture); + const dir = join(fixture.dir, "transcripts"); + const runner = streamingRunner(false); + await runCompare({ config, runners: { baseline: runner, proposed: runner }, transcriptOut: { dir } }); + + // Same naming convention as raw results, one NDJSON line per event. + const baselineLines = readFileSync(join(dir, "case-one_baseline_1.stream.jsonl"), "utf8").trim().split("\n"); + expect(baselineLines).toHaveLength(3); + expect(JSON.parse(baselineLines[0] ?? "{}").arm).toBe("baseline"); + expect(JSON.parse(baselineLines[1] ?? "{}").usage.cache_read_input_tokens).toBe(900); + expect(existsSync(join(dir, "case-one_proposed_1.stream.jsonl"))).toBe(true); + + const measureDir = join(fixture.dir, "transcripts-measure"); + await runMeasure({ config, runner, transcriptOut: { dir: measureDir } }); + expect(existsSync(join(measureDir, "case-one_measure_1.stream.jsonl"))).toBe(true); + + // A run that dies mid-stream keeps what it printed — that partial record is + // what a timeout or a budget abort leaves to investigate. + const failedDir = join(fixture.dir, "transcripts-failed"); + await expect( + runMeasure({ config, runner: streamingRunner(true), transcriptOut: { dir: failedDir } }), + ).rejects.toThrow("killed mid-run"); + expect(readFileSync(join(failedDir, "case-one_measure_1.stream.jsonl"), "utf8").trim().split("\n")).toHaveLength(2); + + // Without the flag nothing is captured and no sink reaches the runner. + let sinkSeen = true; + await runMeasure({ + config, + runner: { + name: "mock", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }, + async run(options: RunnerRunOptions) { + sinkSeen = options.onStreamEvent !== undefined; + return { output: "ok", costUsd: 0, turns: 1, durationMs: 1, models: [], raw: {} }; + }, + }, + }); + expect(sinkSeen).toBe(false); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); + +test("transcriptOut is refused before any paid run when the runner cannot stream events", async () => { + const fixture = fixtureDir(); + try { + let ran = false; + const runner: Runner = { + name: "openai", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, + async run() { + ran = true; + return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: [], raw: {} }; + }, + }; + + const config = cappedConfig(fixture); + // Empty transcript files would misrepresent a runner that simply cannot + // produce one, so this fails loudly instead. + await expect( + runCompare({ + config, + runners: { baseline: runner, proposed: runner }, + transcriptOut: { dir: join(fixture.dir, "transcripts") }, + }), + ).rejects.toThrow('runner "openai" does not'); + expect(ran).toBe(false); + + // The same config without the flag still runs on that runner. + const summary = await runCompare({ config, runners: { baseline: runner, proposed: runner } }); + expect(summary.cases[0]?.baseline.passes).toBe(1); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); + +test("an unwritable transcriptOut dir is refused before any paid run and leaves no sandbox behind", async () => { + const fixture = fixtureDir(); + try { + let ran = false; + const runner: Runner = { + name: "mock-stream", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }, + async run() { + ran = true; + return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: [], raw: {} }; + }, + }; + + const config = cappedConfig(fixture); + // A regular file where the transcript dir belongs makes mkdirSync throw — + // that must surface before scenario 1 ever prepares a sandbox. + const blockedPath = join(fixture.dir, "blocked-transcripts"); + writeFileSync(blockedPath, "not a directory", "utf8"); + + await expect( + runCompare({ + config, + runners: { baseline: runner, proposed: runner }, + transcriptOut: { dir: blockedPath }, + }), + ).rejects.toThrow(blockedPath); + expect(ran).toBe(false); + // Nothing was prepared, so nothing was orphaned. + expect(existsSync(config.sandboxRoot)).toBe(false); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); + +test("transcriptOut round-trips multi-byte event content exactly", async () => { + const fixture = fixtureDir(); + try { + // CJK, an emoji, and accented Latin each encode to a different number of + // UTF-8 bytes per character. This pins the encoding end to end: the sink + // writes bytes, and what lands on disk has to parse back identically. + const multiByte = "こんにちは 🎉 café"; + const runner: Runner = { + name: "mock-stream", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }, + async run(options: RunnerRunOptions) { + options.onStreamEvent?.(JSON.stringify({ type: "assistant", text: multiByte })); + return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: ["sonnet"], raw: {} }; + }, + }; + + const config = cappedConfig(fixture); + const dir = join(fixture.dir, "transcripts-multibyte"); + await runCompare({ config, runners: { baseline: runner, proposed: runner }, transcriptOut: { dir } }); + + const lines = readFileSync(join(dir, "case-one_baseline_1.stream.jsonl"), "utf8").trim().split("\n"); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0] ?? "{}").text).toBe(multiByte); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); + +test("transcriptOut writes a large event line whole", async () => { + const fixture = fixtureDir(); + try { + // A quarter-megabyte line, well past any real event, so the write path is + // exercised at a size where a short write is at least possible. On a + // regular file one writeSync still completes it here, so this is + // round-trip coverage, not proof that the loop handles a short count. + const big = "x".repeat(250_000); + const runner: Runner = { + name: "mock-stream", + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }, + async run(options: RunnerRunOptions) { + options.onStreamEvent?.(JSON.stringify({ type: "assistant", text: big })); + return { output: "ok", costUsd: 0.1, turns: 1, durationMs: 10, models: ["sonnet"], raw: {} }; + }, + }; + + const config = cappedConfig(fixture); + const dir = join(fixture.dir, "transcripts-large"); + await runCompare({ config, runners: { baseline: runner, proposed: runner }, transcriptOut: { dir } }); + + const raw = readFileSync(join(dir, "case-one_baseline_1.stream.jsonl"), "utf8"); + const lines = raw.trim().split("\n"); + expect(lines).toHaveLength(1); + const line = lines[0] ?? ""; + expect(Buffer.byteLength(line, "utf8")).toBe(line.length); // ASCII payload: bytes == chars, sanity-checks the assertion below + expect(JSON.parse(line).text).toBe(big); + } finally { + rmSync(fixture.dir, { recursive: true, force: true }); + } +}); diff --git a/test/delivery.test.ts b/test/delivery.test.ts index 4f72d80..9260320 100644 --- a/test/delivery.test.ts +++ b/test/delivery.test.ts @@ -28,7 +28,7 @@ test("install delivery puts arm skills in each sandbox registry and keeps them o const seenPrompts: string[] = []; const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { seenPrompts.push(options.systemPrompt); expect(options.systemPromptMode).toBe("append"); diff --git a/test/json-grader.test.ts b/test/json-grader.test.ts index 4b0222e..0ca5824 100644 --- a/test/json-grader.test.ts +++ b/test/json-grader.test.ts @@ -209,7 +209,7 @@ test("runCompare grades reasoning prose + trailing JSON with a json grader on a const toolsSeen: string[] = []; const runner: Runner = { name: "openai", - capabilities: { sandboxTools: false, skillRegistry: false, images: false }, + capabilities: { sandboxTools: false, skillRegistry: false, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { toolsSeen.push(options.tools); const isProposed = options.systemPrompt.includes("PROPOSED"); diff --git a/test/judge.test.ts b/test/judge.test.ts index ab7f01a..f64278a 100644 --- a/test/judge.test.ts +++ b/test/judge.test.ts @@ -113,7 +113,7 @@ function mockJudgeFactory(calls: RunnerRunOptions[], factoryArgs: unknown[][] = factoryArgs.push([name, options]); const runner: Runner = { name, - capabilities: { sandboxTools: false, skillRegistry: false, images: false }, + capabilities: { sandboxTools: false, skillRegistry: false, images: false, streamEvents: false }, async run(runOptions: RunnerRunOptions) { calls.push(runOptions); const output = runOptions.userPrompt.includes("GARBLE") @@ -177,7 +177,7 @@ async function expectGateRefusal(tree: JudgeTree, pattern: RegExp): Promise ({ name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run() { return { output, costUsd: 0.1, turns: 1, durationMs: 5, models: ["sonnet"], raw: {} }; }, @@ -299,7 +299,7 @@ test("a judge transport failure fails the graded run with the error in the messa try { setJudgeRunnerFactory(() => ({ name: "claude-p", - capabilities: { sandboxTools: false, skillRegistry: false, images: false }, + capabilities: { sandboxTools: false, skillRegistry: false, images: false, streamEvents: false }, async run(): Promise { throw new Error("endpoint unreachable"); }, diff --git a/test/measure.test.ts b/test/measure.test.ts index 08a56ef..59db4ce 100644 --- a/test/measure.test.ts +++ b/test/measure.test.ts @@ -11,7 +11,7 @@ function mockRunner(outputs: string[]): { runner: Runner; seen: RunnerRunOptions let call = 0; const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { seen.push(options); const output = outputs[call % outputs.length]; diff --git a/test/openai-runner.test.ts b/test/openai-runner.test.ts index 1aa01c0..aead0c0 100644 --- a/test/openai-runner.test.ts +++ b/test/openai-runner.test.ts @@ -101,11 +101,11 @@ test("OpenAiCompatRunner surfaces HTTP errors and malformed completions", async test("createRunner builds runners with the right capabilities", () => { const claude = createRunner("claude-p"); expect(claude.name).toBe("claude-p"); - expect(claude.capabilities).toEqual({ sandboxTools: true, skillRegistry: true, images: false }); + expect(claude.capabilities).toEqual({ sandboxTools: true, skillRegistry: true, images: false, streamEvents: true }); const openai = createRunner("openai", { baseUrl: "http://localhost:8080/v1" }); expect(openai.name).toBe("openai"); - expect(openai.capabilities).toEqual({ sandboxTools: false, skillRegistry: false, images: true }); + expect(openai.capabilities).toEqual({ sandboxTools: false, skillRegistry: false, images: true, streamEvents: false }); }); test("buildChatRequest attaches images as data-URI content parts before the text", () => { diff --git a/test/receipt.test.ts b/test/receipt.test.ts index 59d4430..3a1f836 100644 --- a/test/receipt.test.ts +++ b/test/receipt.test.ts @@ -15,7 +15,7 @@ function sha256(content: string): string { function mockRunner(decide: (options: RunnerRunOptions) => string): Runner { return { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { return { output: decide(options), costUsd: 0.1, turns: 1, durationMs: 5, models: ["m"], raw: {} }; }, diff --git a/test/render.test.ts b/test/render.test.ts index 5be1c6a..47a86c8 100644 --- a/test/render.test.ts +++ b/test/render.test.ts @@ -141,7 +141,7 @@ test("runCompare renders per-case vars into both arms and fails unbound before a const seenPrompts: Array<{ system: string; user: string }> = []; const runner: Runner = { name: "mock", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run(options: RunnerRunOptions) { seenPrompts.push({ system: options.systemPrompt, user: options.userPrompt }); return { output: "ok", costUsd: 0, turns: 1, durationMs: 1, models: ["m"], raw: {} }; diff --git a/test/runner-support.test.ts b/test/runner-support.test.ts index 49381d4..75c931e 100644 --- a/test/runner-support.test.ts +++ b/test/runner-support.test.ts @@ -5,7 +5,7 @@ import type { Runner } from "../src/types"; const textOnlyRunner: Runner = { name: "openai", - capabilities: { sandboxTools: false, skillRegistry: false, images: true }, + capabilities: { sandboxTools: false, skillRegistry: false, images: true, streamEvents: false }, async run() { throw new Error("unused"); }, @@ -56,7 +56,7 @@ test("install delivery is rejected on a runner without a skill registry", () => test("mixed arms validate per arm: the openai arm rejects command graders, text graders pass both", () => { const toolRunner: Runner = { name: "claude-p", - capabilities: { sandboxTools: true, skillRegistry: true, images: false }, + capabilities: { sandboxTools: true, skillRegistry: true, images: false, streamEvents: false }, async run() { throw new Error("unused"); }, diff --git a/test/runner.test.ts b/test/runner.test.ts index 261085c..00c9a08 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -236,3 +236,320 @@ test("non-zero exits without a turn-cap subtype still throw", async () => { rmSync(dir, { recursive: true, force: true }); } }); + +test("buildClaudeArgs switches to stream-json only when a transcript sink is attached", () => { + const base = { + systemPrompt: "system", + systemPromptFile: "/tmp/system.md", + userPrompt: "do work", + model: "sonnet", + cwd: "/tmp/sandbox", + addDirs: [], + tools: "", + timeoutMs: 1_000, + maxBudgetUsd: 0.25, + }; + + expect(buildClaudeArgs(base)).toContain("json"); + expect(buildClaudeArgs(base)).not.toContain("stream-json"); + expect(buildClaudeArgs(base)).not.toContain("--verbose"); + + const streaming = buildClaudeArgs({ ...base, onStreamEvent: () => {} }); + const i = streaming.indexOf("--output-format"); + expect(streaming[i + 1]).toBe("stream-json"); + // stream-json is rejected under -p without --verbose. + expect(streaming).toContain("--verbose"); +}); + +// Captured shape of a stream-json run: NDJSON events, terminal `result` last. +const STREAM_EVENTS = [ + { type: "system", subtype: "init", session_id: "s-1" }, + { type: "assistant", message: { usage: { input_tokens: 12, cache_read_input_tokens: 9_000 } } }, + { type: "assistant", message: { usage: { input_tokens: 40, cache_read_input_tokens: 9_400 } } }, + { + type: "result", + subtype: "success", + result: "done", + num_turns: 2, + total_cost_usd: 0.03, + duration_ms: 4_200, + modelUsage: { "claude-sonnet-5": { costUSD: 0.03 } }, + }, +]; + +function fakeClaude(dir: string, body: string): string { + const bin = join(dir, "fake-claude"); + writeFileSync(bin, `#!/bin/sh\n${body}\n`, { mode: 0o755 }); + return bin; +} + +const streamRunOptions = (cwd: string, onStreamEvent: (line: string) => void) => ({ + systemPrompt: "system", + userPrompt: "do work", + model: "sonnet", + cwd, + addDirs: [] as string[], + tools: "", + timeoutMs: 10_000, + maxBudgetUsd: 1, + onStreamEvent, +}); + +test("stream mode feeds every event to the sink and scores the terminal result event", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + const echoes = STREAM_EVENTS.map((event) => `echo '${JSON.stringify(event)}'`).join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, echoes)); + + const lines: string[] = []; + const result = await runner.run(streamRunOptions(dir, (line) => lines.push(line))); + + // Per-turn usage is the whole point: the aggregate result cannot show how + // context grew between turn 1 and turn 2. + expect(lines).toHaveLength(4); + expect(JSON.parse(lines[1] ?? "{}").message.usage.cache_read_input_tokens).toBe(9_000); + expect(JSON.parse(lines[2] ?? "{}").message.usage.cache_read_input_tokens).toBe(9_400); + + expect(result.output).toBe("done"); + expect(result.turns).toBe(2); + expect(result.costUsd).toBeCloseTo(0.03); + expect(result.models).toEqual(["claude-sonnet-5"]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a truncated stream keeps its lines but has no outcome to score", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // Two whole events then a killed mid-line write: no terminal result event. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${JSON.stringify(STREAM_EVENTS[1])}'`, + `printf '%s' '{"type":"assistant","mess'`, + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + await expect(runner.run(streamRunOptions(dir, (line) => lines.push(line)))).rejects.toThrow( + "no terminal result event", + ); + + // "Last line is the result" is not safe, but the partial stream is still + // the evidence a token-economics run was after. + expect(lines).toHaveLength(3); + expect(lines[2]).toBe('{"type":"assistant","mess'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +// A result-shaped event that is NOT the real terminal one, the kind a +// subagent or compaction can emit mid-stream. Its "result" text is +// distinctive so a test would notice if it were ever scored as the outcome. +const MID_STREAM_RESULT_EVENT = { + type: "result", + subtype: "success", + result: "mid-stream, not the real outcome", + num_turns: 1, + total_cost_usd: 0.01, +}; + +test("a mid-stream result event is not scored as the outcome when the run then exits non-zero", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // A result event fires mid-stream, then the process dies (exit 1) without + // ever emitting the real terminal event. A result event is only trusted + // as terminal when the process also exited cleanly, so this must reject + // rather than return MID_STREAM_RESULT_EVENT as a scored success. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${JSON.stringify(MID_STREAM_RESULT_EVENT)}'`, + "exit 1", + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + await expect(runner.run(streamRunOptions(dir, (line) => lines.push(line)))).rejects.toThrow(/claude exited 1/); + + // The partial stream is still evidence even though the run is scored as a failure. + expect(lines).toHaveLength(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a mid-stream result event is not scored as the outcome when the run is then killed by a signal", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // Same scenario, but the process dies from a signal (as the timeout path's + // SIGTERM/SIGKILL would deliver) instead of a self-chosen exit code. A + // signaled process does not exit 0, so this must reject too. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${JSON.stringify(MID_STREAM_RESULT_EVENT)}'`, + "kill -TERM $$", + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + await expect(runner.run(streamRunOptions(dir, (line) => lines.push(line)))).rejects.toThrow(/claude exited/); + + expect(lines).toHaveLength(2); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the last result event wins when the run exits cleanly", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // A result event fires mid-stream, then the real terminal result event + // follows, and the process exits 0. The mid-stream event must not win + // just because it arrived first — only the last one is the outcome. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${JSON.stringify(MID_STREAM_RESULT_EVENT)}'`, + `echo '${JSON.stringify(STREAM_EVENTS[3])}'`, + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + const result = await runner.run(streamRunOptions(dir, (line) => lines.push(line))); + + expect(lines).toHaveLength(3); + expect(result.output).toBe("done"); + expect(result.turns).toBe(2); + expect(result.costUsd).toBeCloseTo(0.03); + expect(result.output).not.toBe(MID_STREAM_RESULT_EVENT.result); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a mid-stream result event is not scored as the outcome when the run then exits cleanly with no real terminal event", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // A result event fires mid-stream, another event follows it, and then the + // process exits 0 without ever emitting a real terminal result event. + // Claude exiting 0 normally means the run finished cleanly on its result + // event, so this combination is self-contradictory — it must reject + // rather than score MID_STREAM_RESULT_EVENT as the outcome. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${JSON.stringify(MID_STREAM_RESULT_EVENT)}'`, + `echo '${JSON.stringify(STREAM_EVENTS[1])}'`, + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + // A distinct message from the truncated-stream case: a result event was + // produced here, it just was not last, and someone reading the failure + // should not go looking for a missing event. + await expect(runner.run(streamRunOptions(dir, (line) => lines.push(line)))).rejects.toThrow( + /result event before the end of the stream/, + ); + + // The printed lines are still evidence even though the run has no outcome to score. + expect(lines).toHaveLength(3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("stream mode decodes turn caps and budget aborts out of NDJSON", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + const capped = fakeClaude( + dir, + [`echo '${JSON.stringify(STREAM_EVENTS[0])}'`, `echo '${MAX_TURNS_STDOUT}'`, "exit 1"].join("\n"), + ); + const cappedResult = await new ClaudePrintRunner(capped).run(streamRunOptions(dir, () => {})); + expect(cappedResult.exhaustedTurns).toBe(true); + expect(cappedResult.turns).toBe(25); + + const broke = fakeClaude( + dir, + [`echo '${JSON.stringify(STREAM_EVENTS[0])}'`, `echo '${BUDGET_ABORT_STDOUT}'`, "exit 1"].join("\n"), + ); + // Whole-stdout JSON.parse fails on NDJSON, which would have made a budget + // abort read as a bare exit code again. + await expect(new ClaudePrintRunner(broke).run(streamRunOptions(dir, () => {}))).rejects.toThrow("max budget"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a mid-stream turn-cap event followed by more stream and a crash is not scored as exhaustedTurns", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // MAX_TURNS_STDOUT fires mid-stream (not as the last line), another event + // follows it, and then the process exits non-zero without ever emitting a + // real terminal result. The run must be reported as a failure — scoring it + // as exhaustedTurns would corrupt the measurement by turning a crash into + // a counted, "completed" data point. + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `echo '${MAX_TURNS_STDOUT}'`, + `echo '${JSON.stringify(STREAM_EVENTS[1])}'`, + "exit 1", + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + await expect(runner.run(streamRunOptions(dir, (line) => lines.push(line)))).rejects.toThrow(/claude exited 1/); + + expect(lines).toHaveLength(3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a single event line spanning many stdout chunks is still read whole", async () => { + const dir = mkdtempSync(join(tmpdir(), "promptdiff-runner-test-")); + try { + // A pipe hands back far less than 2MB per read, so a field this large + // splits the NDJSON line across many stdout chunks (11 here) — the case + // the per-chunk scan-offset arithmetic has to get right. Built with + // head/tr rather than an embedded literal: a 2,000,000-character + // argument to echo risks blowing past ARG_MAX. + const bigFieldChars = 2_000_000; + const body = [ + `echo '${JSON.stringify(STREAM_EVENTS[0])}'`, + `printf '%s' '{"type":"assistant","big":"'`, + `head -c ${bigFieldChars} /dev/zero | tr '\\0' 'a'`, + `printf '"}\\n'`, + // A short line after the giant one: if a late chunk happens to land + // two newlines at once (the giant line's close plus this line's own), + // a stale scan offset would merge them into one garbled line instead + // of missing them outright — this line is what would expose that. + // It comes before the terminal result event (not after): a result + // event only counts as the run's outcome when it is the last line the + // stream produced, so this line has to sit ahead of it, not behind it. + `echo '${JSON.stringify(STREAM_EVENTS[1])}'`, + `echo '${JSON.stringify(STREAM_EVENTS[3])}'`, + ].join("\n"); + const runner = new ClaudePrintRunner(fakeClaude(dir, body)); + + const lines: string[] = []; + const result = await runner.run(streamRunOptions(dir, (line) => lines.push(line))); + + // Exactly the four lines the script printed: init, the giant line, the + // short line, and the terminal result — a dropped, duplicated, or + // merged newline boundary would show up here as the wrong count. + expect(lines).toHaveLength(4); + + const big = JSON.parse(lines[1] ?? "{}") as { big: string }; + expect(big.big).toHaveLength(bigFieldChars); + expect(big.big).toBe("a".repeat(bigFieldChars)); + + const trailing = JSON.parse(lines[2] ?? "{}") as { message: { usage: { cache_read_input_tokens: number } } }; + expect(trailing.message.usage.cache_read_input_tokens).toBe(9_000); + + expect(result.output).toBe("done"); + expect(result.turns).toBe(2); + expect(result.costUsd).toBeCloseTo(0.03); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +});