From fc429e4ef26128af16cc5741c790f6cd23976fd4 Mon Sep 17 00:00:00 2001 From: Uliseis Date: Mon, 9 Nov 2026 21:15:41 +0100 Subject: [PATCH] fix(report): terminal and markdown output remove control characters from run values Values that an agent, a simulated caller, a platform's call or a results file holds reach the terminal, report --markdown and LEADERBOARD.md as text. C0, DEL and C1 controls and the bidi embedding, override and isolate controls become spaces, before anything is cut to fit, so no escape sequence, bell or reversed text gets through. sandtable's own colours are added after and stay. src/report/printable.ts holds the helpers: printable for a value on one line, printableLines for a message that keeps its own lines, and printableCopy for a run or a calibration that a renderer prints from. results.json, the HTML reports and every contract are unchanged. --- CHANGELOG.md | 4 + src/cli.ts | 98 ++++----- src/judge/calibrate.ts | 12 +- src/report/markdown.ts | 6 +- src/report/printable.ts | 33 +++ src/report/terminal.ts | 26 ++- src/run/gates.ts | 3 +- src/run/leaderboard.ts | 6 +- src/shell.ts | 13 ++ test/control-characters.test.ts | 354 ++++++++++++++++++++++++++++++++ test/judge.test.ts | 10 + test/serve-driven.test.ts | 27 +++ 12 files changed, 526 insertions(+), 66 deletions(-) create mode 100644 src/report/printable.ts create mode 100644 test/control-characters.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index c91b7a59..85140109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/2.0.0/), ## [Unreleased] +### Security + +- Terminal output, `report --markdown` and `LEADERBOARD.md`: control characters, and the bidi controls that show text reversed, now show as spaces in every value that comes from an agent, a simulated caller, a platform's call, a results file, a transcript's file name or a domain that `init` drafted, and a command sandtable suggests quotes such a file name so the shell still reads it right. Before, such text could write escape sequences to the terminal of whoever ran `run`, `serve`, `compare`, `regrade`, `leaderboard`, `review`, `judge` or `calibrate`, and set the window title, clear the screen or hide the output that follows. Upgrade if you run `serve` behind a tunnel or read results files you didn't write ([GHSA-qrx5-3qq7-7w59](https://github.com/sandtable-ai/sandtable/security/advisories/GHSA-qrx5-3qq7-7w59)). + ## [1.5.0] - 2026-11-09 A minor release with no breaking changes, and a security fix in the HTML reports: upgrade if you open reports built from domains or results files you didn't write. `sandtable judge` and `run --judge` score a run on judged criteria with a model, apart from pass^k, which they never change, and `sandtable calibrate` measures a judge against human labels. Clock dates reach the policy, param defaults, fault messages and `reply_includes`, and `report --markdown` writes a summary for a CI job or a pull request comment. diff --git a/src/cli.ts b/src/cli.ts index 6c6d5d02..ea9f53d4 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { draftUsageLines, formatPercent, renderCompare, renderJudged, renderJudg import { renderHtml } from "./report/html.ts"; import { renderCustomerHtml } from "./report/customer.ts"; import { renderMarkdown } from "./report/markdown.ts"; +import { printable, printableLines } from "./report/printable.ts"; import { everyTrialFailed, runSuite, trialStatus, type SuiteResult } from "./run/suite.ts"; import { runGates } from "./run/gates.ts"; import { startWorldServer } from "./serve/server.ts"; @@ -253,7 +254,7 @@ function selectTasks(domain: Domain, taskIds: string[] | undefined, tags: string function failIfNothingGraded(result: SuiteResult): boolean { const error = everyTrialFailed(result); if (error === undefined) return false; - console.error(`\n✗ Every trial ended in an error, so this run says nothing about the agent. The first error:\n ${error}`); + console.error(`\n✗ Every trial ended in an error, so this run says nothing about the agent. The first error:\n ${printable(error)}`); exitWith(EXIT_ERROR); return true; } @@ -274,7 +275,7 @@ function failIfRegressed(comparison: Comparison): void { const broken = regressions(comparison); if (broken.length === 0) return; const counts = (x?: { passes: number; trials: number }) => (x ? `${x.passes}/${x.trials}` : "-"); - console.error(`\n✗ ${broken.length} task(s) broke since the baseline (--fail-on-regression): ${broken.map((d) => `${d.id} ${counts(d.before)} → ${counts(d.after)}`).join(", ")}`); + console.error(`\n✗ ${broken.length} task(s) broke since the baseline (--fail-on-regression): ${broken.map((d) => `${printable(d.id)} ${counts(d.before)} → ${counts(d.after)}`).join(", ")}`); exitWith(EXIT_GATE); } @@ -287,7 +288,7 @@ async function loadForCheck(domainPath: string): Promise { return await loadDomain(await resolveDomain(domainPath)); } catch (err) { if (!(err instanceof DomainError) || err instanceof DomainNotFoundError) throw err; - console.error(`Invalid domain: ${err.message}`); + console.error(printableLines(`Invalid domain: ${err.message}`)); exitWith(EXIT_GATE); return undefined; } @@ -383,7 +384,7 @@ function callDateLines(transcripts: { file: string; text: string }[], fallback: "Call dates, which each chat's drafted dates are made relative to:", ...transcripts.map((t) => { const stated = callDateIn(t.text); - return ` ${t.file}: ${stated !== undefined ? `${stated}, stated in the file` : fallback !== undefined ? `${fallback}, from --call-date` : "none: its dates stay literal"}`; + return ` ${printable(t.file)}: ${stated !== undefined ? `${stated}, stated in the file` : fallback !== undefined ? `${fallback}, from --call-date` : "none: its dates stay literal"}`; }), ]; } @@ -493,13 +494,13 @@ withProviderFlags(program openapiDraft = { draft, files }; if (o.dryRun) { draftedDomainChars = JSON.stringify(YAML.parse(files[DOMAIN_FILE]!)).length + files["policy.md"].length; - console.log(`Would draft ${draft.name} from ${o.openapi}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools. No model call.`); + console.log(`Would draft ${printable(draft.name)} from ${o.openapi}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools. No model call.`); for (const [f, text] of Object.entries(files)) { const target = path.join(o.out, f); const exists = await stat(target).then(() => true, () => false); console.log(` would write ${target}${exists && !o.force ? " (exists: init stops unless you pass --force)" : ""} (${text.length} characters)`); } - for (const w of draft.warnings) console.log(` ! ${w}`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); } } if (o.dryRun) { @@ -518,8 +519,8 @@ withProviderFlags(program const { draft, files } = openapiDraft; const written = await writeDraft(o.out, files, o.force); const notes = draft.tools.reduce((n, t) => n + t.notes.length, 0); - console.log(`Drafted ${draft.name}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools, ${notes} review notes.`); - for (const w of draft.warnings) console.log(` ! ${w}`); + console.log(`Drafted ${printable(draft.name)}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools, ${notes} review notes.`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); for (const f of written) console.log(` wrote ${f}`); } if (sopText !== undefined || transcripts) { @@ -544,7 +545,7 @@ withProviderFlags(program const file = await appendToDomainList(o.out, "invariants", res.invariants, softRuleComments(res.softRules)); console.log(` ${res.invariants.length} invariant(s) drafted (review: pending), ${res.softRules.length} soft rule(s) noted in ${file}`); for (const l of draftUsageLines(res.usage)) console.log(l); - for (const w of res.warnings) console.log(` ! ${w}`); + for (const w of res.warnings) console.log(` ! ${printable(w)}`); } // Transcripts that drafted nothing: those whose call failed, and those never sent once a failure stopped drafting; // `all` when that is every transcript. @@ -571,7 +572,7 @@ withProviderFlags(program } if (res.literal.length) console.log(` ! ${res.literal.length} task(s) keep literal dates, since their chat states no call date: pass --call-date, or see their TODO(review) notes`); for (const l of draftUsageLines(res.usage)) console.log(l); - for (const w of res.warnings) console.log(` ! ${w}`); + for (const w of res.warnings) console.log(` ! ${printable(w)}`); if (res.failed.length) undrafted = { failed: res.failed, notSent: res.notSent, all: res.failed.length + res.notSent.length === transcripts.length }; } // Nothing to review when the transcripts were the only step and none of them drafted. @@ -588,7 +589,7 @@ withProviderFlags(program if (notSent.length) { // An error every later call would hit too stopped drafting: it is named once, at the chat it hit. const sent = notSent.length === 1 ? "1 transcript wasn't sent" : `${notSent.length} transcripts weren't sent`; - console.error(`\n✗ Drafting stopped at ${failed.at(-1)!.file}: its call failed in a way every later call would too, so ${sent}. ${all ? "Nothing was drafted from the transcripts:" : others}`); + console.error(`\n✗ Drafting stopped at ${printable(failed.at(-1)!.file)}: its call failed in a way every later call would too, so ${sent}. ${all ? "Nothing was drafted from the transcripts:" : others}`); } else { console.error( all @@ -596,8 +597,8 @@ withProviderFlags(program : `\n✗ ${failed.length} of ${n} transcripts' drafting calls failed, so nothing was drafted from ${failed.length === 1 ? "it" : "them"}. ${others}`, ); } - for (const f of failed) console.error(` ${f.file}: ${f.error}`); - if (notSent.length) console.error(` not sent: ${notSent.join(", ")}`); + for (const f of failed) console.error(` ${printable(f.file)}: ${printable(f.error)}`); + if (notSent.length) console.error(` not sent: ${notSent.map(printable).join(", ")}`); const paths = all ? [o.transcripts] : [...failed.map((f) => f.file), ...notSent].map((file) => transcripts.find((t) => t.file === file)!.path); console.error(` Draft ${left === 1 ? "it" : "them"} again with:`); for (const p of paths) console.error(` ${transcriptsRerun(p, o)}`); @@ -662,8 +663,8 @@ program const rendered = importer.render(draft); const written = await writeDraft(o.out, { [DOMAIN_FILE]: rendered.domain, "policy.md": draft.policy, "tasks.yaml": rendered.tasks }, o.force); const criteria = draft.tasks.reduce((n, t) => n + (t.judged?.length ?? 0), 0); - console.log(`Imported ${draft.name} from ${importer.label}: ${draft.tools.length} tools, ${Object.keys(draft.tables).length} tables, ${draft.tasks.length} tasks, ${criteria} judged ${criteria === 1 ? "criterion" : "criteria"}, all pending review.`); - for (const w of draft.warnings) console.log(` ! ${w}`); + console.log(`Imported ${printable(draft.name)} from ${importer.label}: ${draft.tools.length} tools, ${Object.keys(draft.tables).length} tables, ${draft.tasks.length} tasks, ${criteria} judged ${criteria === 1 ? "criterion" : "criteria"}, all pending review.`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); for (const f of written) console.log(` wrote ${f}`); console.log( `\nNext: accept or fix each drafted item, draft the rules from the prompt, and validate:\n ${NEXT} review ${shellWord(o.out)}\n ${NEXT} init --sop ${shellWord(path.join(o.out, "policy.md"))} -o ${shellWord(o.out)} --dry-run\n ${NEXT} validate ${shellWord(o.out)} --strict\n${importer.serve(o.out)}`, @@ -678,8 +679,8 @@ program .action(async (domainPath: string, o: { strict?: boolean }) => { const d = await loadForCheck(domainPath); if (!d) return; - console.log(`✓ ${d.name}: ${Object.keys(d.world.tables).length} tables, ${d.tools.length} tools, ${d.invariants.length} invariants, ${d.tasks.length} tasks`); - for (const w of domainWarnings(d)) console.log(`⚠ ${w}`); + console.log(`✓ ${printable(d.name)}: ${Object.keys(d.world.tables).length} tables, ${d.tools.length} tools, ${d.invariants.length} invariants, ${d.tasks.length} tasks`); + for (const w of domainWarnings(d)) console.log(`⚠ ${printableLines(w)}`); const pending = pendingReview(d); if (pending.length) { console.log(`⚠ ${pending.length} drafted item(s) pending review (${NEXT} review ${shellWord(domainPath)})`); @@ -699,15 +700,15 @@ program const solvable = t.solvable === null ? "solvable ?" : t.solvable ? "solvable" : "NOT solvable"; const kind = t.expectsWrite ? "write" : "no-write"; const mark = t.errors.length ? "✗" : t.warnings.length ? "!" : "✓"; - console.log(`${mark} ${t.id} (${kind} · ${t.ready ? "ready" : "NOT ready"} · ${solvable} · noop ${t.noopPasses ? "passes" : "fails"})`); - for (const e of t.errors) console.log(` ✗ ${e}`); - for (const w of t.warnings) console.log(` ! ${w}`); + console.log(`${mark} ${printable(t.id)} (${kind} · ${t.ready ? "ready" : "NOT ready"} · ${solvable} · noop ${t.noopPasses ? "passes" : "fails"})`); + for (const e of t.errors) console.log(` ✗ ${printableLines(e)}`); + for (const w of t.warnings) console.log(` ! ${printableLines(w)}`); } // Rules no task puts in play: a green run says nothing about them. A tool no task uses is a note, not a warning. const idle = report.rules.filter((r) => r.warnings.length); if (report.rules.length) console.log(`\n${report.rules.length - idle.length} of ${report.rules.length} rules in play in at least one task.`); - for (const r of idle) for (const w of r.warnings) console.log(`! ${r.id}: ${w}`); - if (report.unusedTools.length) console.log(`Tools no task uses: ${report.unusedTools.join(", ")}`); + for (const r of idle) for (const w of r.warnings) console.log(`! ${printable(r.id)}: ${printableLines(w)}`); + if (report.unusedTools.length) console.log(`Tools no task uses: ${report.unusedTools.map(printable).join(", ")}`); const noWrite = report.tasks.filter((t) => !t.expectsWrite).length; const noopPasses = report.tasks.filter((t) => t.noopPasses).length; console.log(`\n${report.tasks.length} tasks: ${report.tasks.length - noWrite} expect a write, ${noWrite} do not; a do-nothing agent passes ${noopPasses} of ${report.tasks.length}.`); @@ -726,9 +727,10 @@ program // The notes `init` left above each item, such as a status code to encode or a value copied from a chat. const notes = await reviewNotes(target); for (const p of pending) { - console.log(`- ${p.kind} ${p.id}${p.source ? ` (${p.source.file})` : ""}`); - if (p.source?.quote) console.log(` "${p.source.quote}"`); - for (const n of notes.get(`${p.kind} ${p.id}`) ?? []) console.log(` TODO(review): ${n}`); + console.log(`- ${p.kind} ${printable(p.id)}${p.source ? ` (${printable(p.source.file)})` : ""}`); + // A quote can be a caller's words from a transcript, and a note a model's: text, never escape sequences. + if (p.source?.quote) console.log(` "${printableLines(p.source.quote)}"`); + for (const n of notes.get(`${p.kind} ${p.id}`) ?? []) console.log(` TODO(review): ${printable(n)}`); } console.log(`\nAccept an item by changing its \`review: pending\` to \`review: accepted\` (or deleting the line).`); }); @@ -756,11 +758,11 @@ function judgeCommand(results: string, domainPath: string, o: RunJudgeFlags): st */ async function judgeFinishedRun(result: SuiteResult, domain: Domain, domainPath: string, out: string, o: RunJudgeFlags): Promise { if (!statesCriteria(domain, result)) { - console.log(`\n${domain.name} states no judged criteria for the tasks of this run, so there is nothing to judge.`); + console.log(`\n${printable(domain.name)} states no judged criteria for the tasks of this run, so there is nothing to judge.`); return; } const failure = (what: string) => - console.error(`\n✗ ${what}\n The run's results are saved, and its gates alone decide the exit code. To judge it: ${judgeCommand(path.join(out, "results.json"), domainPath, o)}`); + console.error(`\n✗ ${printableLines(what)}\n The run's results are saved, and its gates alone decide the exit code. To judge it: ${judgeCommand(path.join(out, "results.json"), domainPath, o)}`); try { const claude = { ...providerOpts(o), model: o.judgeModel ?? DEFAULT_JUDGE_MODEL }; const plan = judgePlan(domain, result, claude); @@ -777,14 +779,14 @@ async function judgeFinishedRun(result: SuiteResult, domain: Domain, domainPath: claude, concurrency: o.concurrency, onTrial: ({ trial, result: judged, warnings }) => { - const label = `${trial.taskId} trial ${trial.trial + 1}`; + const label = `${printable(trial.taskId)} trial ${trial.trial + 1}`; const tally = (verdict: string, word: string) => { const n = trial.judged!.verdicts.filter((v) => v.verdict === verdict).length; return n ? [`${n} ${word}`] : []; }; const verdicts = [...tally("pass", "pass"), ...tally("fail", "fail"), ...tally("not_applicable", "not applicable")].join(", "); - console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${trial.judged!.error}` : ` ${label}: ${verdicts || "no verdict"}`); - for (const w of warnings) console.log(` ! ${w}`); + console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${printable(trial.judged!.error)}` : ` ${label}: ${verdicts || "no verdict"}`); + for (const w of warnings) console.log(` ! ${printable(w)}`); writing = writing.then(() => save(judged, out, false).then(() => undefined)); return writing; }, @@ -883,7 +885,7 @@ withProviderFlags(program ...(server ? { server } : {}), onTrial: (r) => { done++; - if (process.stderr.isTTY) process.stderr.write(`\r ${done}/${total} trials ${STATUS_MARK[trialStatus(r)]} ${r.taskId}`.padEnd(70)); + if (process.stderr.isTTY) process.stderr.write(`\r ${done}/${total} trials ${STATUS_MARK[trialStatus(r)]} ${printable(r.taskId)}`.padEnd(70)); }, }).finally(() => server?.stop()); if (process.stderr.isTTY) process.stderr.write("\r".padEnd(72) + "\r"); @@ -939,11 +941,11 @@ program label: o.label, idleTimeout: o.idleTimeout, ...(task ? { task } : {}), - onOpen: (s) => console.log(` + ${s.id} opened: a trial of ${s.task.id}`), + onOpen: (s) => console.log(` + ${s.id} opened: a trial of ${printable(s.task.id)}`), onGraded: (trial, result) => { const mark = STATUS_MARK[trialStatus(trial)]; - const failed = trial.checks.filter((c) => !c.passed).map((c) => c.id); - console.log(` ${mark} ${trial.taskId} trial ${trial.trial + 1}${failed.length ? `: ${failed.join(", ")}` : ""}${trial.error ? ` (${trial.error})` : ""}`); + const failed = trial.checks.filter((c) => !c.passed).map((c) => printable(c.id)); + console.log(` ${mark} ${printable(trial.taskId)} trial ${trial.trial + 1}${failed.length ? `: ${failed.join(", ")}` : ""}${trial.error ? ` (${printable(trial.error)})` : ""}`); // One write at a time, in order, so the files always hold the latest grade; the end is answered after it. writing = writing.then(() => save(result, out, o.html).then(() => undefined)).catch((err: unknown) => console.error(` ! could not write ${out}: ${err instanceof Error ? err.message : String(err)}`)); return writing; @@ -954,8 +956,8 @@ program // Before it says it's serving, so no conversation is graded with nowhere to write it. await prepareOutFolder(out, outFolder("serve", o.out)); const url = served.server.url; - console.log(`Serving ${domain.name} at ${url} for conversations another system drives, until you stop it (Ctrl-C).`); - console.log(` Each conversation is a trial of ${task ? task.id : "the task it names (POST /sessions with { \"task\": \"\" })"}: REST ${url}/tools/, MCP ${url}/mcp, webhooks ${url}/webhook.`); + console.log(`Serving ${printable(domain.name)} at ${url} for conversations another system drives, until you stop it (Ctrl-C).`); + console.log(` Each conversation is a trial of ${task ? printable(task.id) : "the task it names (POST /sessions with { \"task\": \"\" })"}: REST ${url}/tools/, MCP ${url}/mcp, webhooks ${url}/webhook.`); console.log(` A conversation ends with POST ${url}/sessions//end, a Vapi end-of-call report, Retell's call_ended or an MCP DELETE, and is graded then.`); if (o.idleTimeout > 0) { console.log(` One with no request for ${o.idleTimeout} minute${o.idleTimeout === 1 ? "" : "s"} is ended without its end, and skipped unless the agent broke a rule (--idle-timeout).`); @@ -1025,7 +1027,7 @@ withProviderFlags(program ...(uses.claude ? { provider: providerOf(p) } : {}), ...(region ? { region } : {}), }; - console.log(`Leaderboard: ${cfg.models.length} model(s) on ${domain.name} (${selected.length} tasks, k=${k}, caller ${user.name})\n`); + console.log(`Leaderboard: ${cfg.models.length} model(s) on ${printable(domain.name)} (${selected.length} tasks, k=${k}, caller ${user.name})\n`); if (runInfo.sandtable.dirty) console.log(" ! the sandtable checkout has uncommitted changes: run.json records the commit, not them\n"); const mark = { cached: "↺ cached ", regraded: "↻ regraded", ran: "✓ ran ", failed: "✗ failed ", skipped: "- skipped " } as const; const result = await runLeaderboard({ @@ -1041,7 +1043,7 @@ withProviderFlags(program userSpec, ...(prices ? { prices } : {}), resolveAgent: (spec, effort) => leaderboardAgent(spec, effort, p, o.openaiBaseUrl), - onEntry: (label, status, note) => console.log(` ${mark[status]} ${label}${note ? ` (${note})` : ""}`), + onEntry: (label, status, note) => console.log(` ${mark[status]} ${printable(label)}${note ? ` (${printable(note)})` : ""}`), }); console.log(`\n${renderLeaderboardMarkdown(result)}`); console.log(` ${path.join(out, "leaderboard.html")}`); @@ -1191,7 +1193,7 @@ program console.log(renderTerminal(result)); const trials = result.tasks.reduce((n, t) => n + t.trials.length, 0); console.log(`\n regraded ${trials} trial(s) without calling a model: ${flipped.length} verdict(s) changed`); - for (const { before, after } of flipped) console.log(` ${STATUS_MARK[trialStatus(before)]} → ${STATUS_MARK[trialStatus(after)]} ${after.taskId} trial ${after.trial + 1}`); + for (const { before, after } of flipped) console.log(` ${STATUS_MARK[trialStatus(before)]} → ${STATUS_MARK[trialStatus(after)]} ${printable(after.taskId)} trial ${after.trial + 1}`); for (const j of [...new Set(["1", String(result.k)])]) { if (stored.passHat[j] !== result.passHat[j]) console.log(` pass^${j} ${formatPercent(stored.passHat[j] ?? 0)} → ${formatPercent(result.passHat[j] ?? 0)}`); } @@ -1219,12 +1221,12 @@ withProviderFlags(program const blocker = judgeBlocker(domain, stored); if (blocker) throw new Error(`can't judge ${resultsPath}: ${blocker}${stored.domain !== domain.name ? `. Pass the --domain the run used: the one whose sandtable.yaml has \`name: ${stored.domain}\`` : ""}`); if (!statesCriteria(domain, stored)) { - console.log(`${domain.name} states no judged criteria for the tasks of this run, so there is nothing to judge.`); + console.log(`${printable(domain.name)} states no judged criteria for the tasks of this run, so there is nothing to judge.`); return; } const claude = { ...providerOpts(o), model: o.model }; const warning = judgeInputsWarning(domain, stored); - if (warning) console.log(` ! ${warning}`); + if (warning) console.log(` ! ${printableLines(warning)}`); const plan = judgePlan(domain, stored, claude); console.log(judgePlanLine(plan)); if (plan.dropped.length) console.log(` ${droppedLine(plan.dropped)}`); @@ -1248,14 +1250,14 @@ withProviderFlags(program claude, concurrency: o.concurrency, onTrial: ({ trial, result, warnings }) => { - const label = `${trial.taskId} trial ${trial.trial + 1}`; + const label = `${printable(trial.taskId)} trial ${trial.trial + 1}`; const tally = (verdict: string, word: string) => { const n = trial.judged!.verdicts.filter((v) => v.verdict === verdict).length; return n ? [`${n} ${word}`] : []; }; const verdicts = [...tally("pass", "pass"), ...tally("fail", "fail"), ...tally("not_applicable", "not applicable")].join(", "); - console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${trial.judged!.error}` : ` ${label}: ${verdicts || "no verdict"}`); - for (const w of warnings) console.log(` ! ${w}`); + console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${printable(trial.judged!.error)}` : ` ${label}: ${verdicts || "no verdict"}`); + for (const w of warnings) console.log(` ! ${printable(w)}`); writing = writing.then(() => save(result, out, false).then(() => undefined)); return writing; }, @@ -1266,7 +1268,7 @@ withProviderFlags(program throw new Error(`judge stopped: ${outcome.stopped.error}\n${outcome.stopped.notJudged} trial(s) were not judged.${kept}`); } await save(outcome.result, out, o.html); - console.log(`\nsandtable judge · ${outcome.result.domain} · agent ${outcome.result.agent}\n`); + console.log(`\nsandtable judge · ${printable(outcome.result.domain)} · agent ${printable(outcome.result.agent)}\n`); console.log(renderJudged(outcome.result).join("\n")); if (outcome.failed) console.log(`\n ${outcome.failed} of ${outcome.sent} judge call(s) failed, each kept as judged.error on its trial: the judged score leaves them out`); console.log(`\n results: ${results}${o.html ? `\n report: ${path.join(out, "report.html")}` : ""}`); @@ -1282,7 +1284,7 @@ program .action(async (resultsPath: string, o: { labels: string }) => { const results = await readResults(resultsPath, "calibrate"); const calibration = calibrate(results, parseLabels(await readUserText(o.labels, { from: "--labels" }), o.labels)); - console.log(`\nsandtable calibrate · ${results.domain} · agent ${results.agent}\n`); + console.log(`\nsandtable calibrate · ${printable(results.domain)} · agent ${printable(results.agent)}\n`); console.log(renderCalibration(calibration)); }); @@ -1311,6 +1313,8 @@ program }); program.parseAsync().catch((err: unknown) => { - console.error(err instanceof DomainNotFoundError ? err.message : err instanceof DomainError ? `Invalid domain: ${err.message}` : err instanceof Error ? err.message : err); + const message = err instanceof DomainNotFoundError ? err.message : err instanceof DomainError ? `Invalid domain: ${err.message}` : err instanceof Error ? err.message : err; + // An error can quote an agent, a platform, a model's API or a results file: its lines as text, never escape sequences. + console.error(typeof message === "string" ? printableLines(message) : message); process.exit(EXIT_ERROR); }); diff --git a/src/judge/calibrate.ts b/src/judge/calibrate.ts index 0c1a4d9d..a1a84cf9 100644 --- a/src/judge/calibrate.ts +++ b/src/judge/calibrate.ts @@ -1,5 +1,6 @@ import YAML from "yaml"; import { z } from "zod"; +import { printableCopy } from "../report/printable.ts"; import { formatPercent } from "../report/terminal.ts"; import type { TrialResult } from "../run/episode.ts"; import type { SuiteResult, TaskSummary } from "../run/suite.ts"; @@ -220,8 +221,6 @@ const WHY_NONE: Record = { const rateCell = (r: CalibrationRate) => `${r.rate === null ? "n/a" : formatPercent(r.rate)} (n=${r.n})`; const counted = (n: number, one: string, many: string) => `${n} ${n === 1 ? one : many}`; -/** A quote and a reason are a model's output, and a quote comes from the transcript of the agent under test: one line of text, nothing a terminal would act on. */ -const oneLine = (s: string) => s.replace(/[\s\p{Cc}]+/gu, " ").trim(); function rateTable(m: ModelCalibration): string[] { const outside = (c: CalibrationCounts) => [...(c.notApplicable ? [`${c.notApplicable} not applicable`] : []), ...(c.unlabelled ? [`${c.unlabelled} with no label`] : [])].join(", "); @@ -233,13 +232,16 @@ function rateTable(m: ModelCalibration): string[] { function pairLines(p: LabelledVerdict): string[] { return [ ` ${p.task} trial ${p.trial} · ${p.criterion}: you said ${p.label}, the judge said ${p.verdict}`, - ...(p.quote !== undefined ? [` quote: "${oneLine(p.quote)}"${p.quoteFound === false ? " (not in the transcript)" : ""}`] : []), - ` reason: ${oneLine(p.reason)}`, + ...(p.quote !== undefined ? [` quote: "${p.quote}"${p.quoteFound === false ? " (not in the transcript)" : ""}`] : []), + ` reason: ${p.reason}`, ]; } /** The terminal's account of a calibration: the rates with the pairs behind each, what to read, and what stayed outside. */ -export function renderCalibration(c: Calibration): string { +export function renderCalibration(calibration: Calibration): string { + // A quote and a reason are a model's output, a quote comes from the transcript of the agent under test, and the ids and + // models from a results file: each is shown as one line of text, nothing a terminal would act on. + const c = printableCopy(calibration); const lines = [" How often the judge agrees with your labels, over pairs with both a label and a pass or fail verdict.", " TPR: of the pairs you labelled pass, the share the judge gave pass. TNR: the same for fail. No model was called, and this sets no gate."]; if (c.models.length > 1) lines.push("", ` This file's trials hold ${c.models.length} judge models, so each is reported apart: mixing them would measure neither. To compare models, keep one results file per model (judge -m -o ) and calibrate each.`); if (c.models.length === 0) lines.push("", " No trial holds a verdict, so there is nothing to measure yet: run npx sandtable-ai judge again."); diff --git a/src/report/markdown.ts b/src/report/markdown.ts index e7ffa0b7..7ebb113b 100644 --- a/src/report/markdown.ts +++ b/src/report/markdown.ts @@ -1,6 +1,7 @@ import type { JudgedCounts } from "../judge/summary.ts"; import { comparisonOf, type Comparison, type TaskChange } from "../run/compare.ts"; import { skippedTrials, tagsBelowFloor, type SuiteResult, type TaskSummary } from "../run/suite.ts"; +import { printable } from "./printable.ts"; import { formatPercent } from "./terminal.ts"; /** @@ -34,11 +35,12 @@ const clip = (s: string, width = CELL) => (s.length <= width ? s : `${s.slice(0, /** * Text a run holds, on one line, so that it renders as itself: it can't break a table or open a tag or a link, and in a * comment it can't notify anyone, link an issue or a pull request, or set math. A URL stays plain text. GitHub can still - * link a commit hash or a CVE id. + * link a commit hash or a CVE id. Control characters, a newline and a tab among them, and bidi controls are spaces, so + * nothing in it acts on a terminal that prints it or shows reversed. */ function escapeMarkdown(s: unknown): string { // As entities, not backslashes: a renderer that ignores a backslash there would still see a tag. - const entities = clip(String(s).replace(/\s*[\r\n]+\s*/g, " ")) + const entities = clip(printable(s)) .replace(/&/g, "&") .replace(//g, ">"); diff --git a/src/report/printable.ts b/src/report/printable.ts new file mode 100644 index 00000000..7eb95900 --- /dev/null +++ b/src/report/printable.ts @@ -0,0 +1,33 @@ +/* + * Text that an agent, a simulated caller, a platform's call or a results file holds, made fit to print: a terminal shows + * it as text, so it can't start an escape sequence, ring the bell, move the cursor or set the window's title, and a + * renderer can't show it reversed. Only what came from outside goes through here: sandtable's own colours are added + * after, so they stay. + */ + +/** C0 controls (a newline and a tab among them), DEL, C1 controls, and the bidi embedding, override and isolate controls. */ +const CONTROL = "\\p{Cc}\\u202a-\\u202e\\u2066-\\u2069"; +const ONE_LINE = new RegExp(`[\\s${CONTROL}]+`, "gu"); +const EACH = new RegExp(`[${CONTROL}]`, "gu"); +/** One character a terminal acts on rather than shows. */ +export const CONTROL_CHAR = new RegExp(`[${CONTROL}]`, "u"); +const LINES = new RegExp(`(?:(?![\\n\\t])[${CONTROL}])+`, "gu"); + +/** On one line, for a value inside a line or a Markdown cell: each run of whitespace and control characters is one space, with none at either end. */ +export function printable(value: unknown): string { + return String(value).replace(ONE_LINE, " ").trim(); +} + +/** Over several lines, for a message that breaks its own lines, such as an error: newlines and tabs stay, and each other run of control characters is one space. */ +export function printableLines(value: string): string { + return value.replace(/\r\n/g, "\n").replace(LINES, " "); +} + +/** A copy of JSON data, such as a run, in which every string is `printable` and every key has each control character as a space, so keys that differ stay apart. */ +export function printableCopy(data: T): T { + if (typeof data === "string") return printable(data) as T; + if (Array.isArray(data)) return data.map(printableCopy) as T; + // fromEntries, not assignment, so a key such as __proto__ stays a key. + if (data !== null && typeof data === "object") return Object.fromEntries(Object.entries(data).map(([k, v]) => [k.replace(EACH, " "), printableCopy(v)])) as T; + return data; +} diff --git a/src/report/terminal.ts b/src/report/terminal.ts index 82b4053d..41c448c1 100644 --- a/src/report/terminal.ts +++ b/src/report/terminal.ts @@ -3,6 +3,7 @@ import { comparisonOf, type Comparison, type TaskChange } from "../run/compare.t import { gradedOutcomes, skippedTrials, tagsBelowFloor, trialStatus, type SoftScore, type SuiteResult, type TaskSummary } from "../run/suite.ts"; import { costOf, totalTokens, type UsageByModel } from "../run/usage.ts"; import type { JudgedCounts } from "../judge/summary.ts"; +import { printableCopy } from "./printable.ts"; const useColor = process.stdout.isTTY && !process.env.NO_COLOR; const paint = (code: number) => (s: string) => (useColor ? `\x1b[${code}m${s}\x1b[0m` : s); @@ -94,8 +95,8 @@ function taskNote(t: TaskSummary): string { * what the judge's calls used and cost. Empty for a run that was never judged. */ export function renderJudged(r: SuiteResult): string[] { - const j = r.judged; - if (!j) return []; + if (!r.judged) return []; + const j = printableCopy(r.judged); const score = (c: JudgedCounts) => (c.score === null ? "n/a" : formatPercent(c.score)); const tally = (c: JudgedCounts) => [`${c.pass} pass`, `${c.fail} fail`, ...(c.notApplicable ? [`${c.notApplicable} not applicable`] : [])].join(", "); const lines = [bold(" Judged score") + dim(" (a model's judgement: it never changes pass/fail, pass^k, a gate or an exit code)")]; @@ -120,7 +121,11 @@ export function renderJudged(r: SuiteResult): string[] { * file it came from (else the block names the baseline's run id). `withoutJudgedChanges` leaves out how the judged * score changed, for a summary printed before the run is judged. */ -export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withoutJudgedChanges?: boolean } = {}): string { +export function renderTerminal(result: SuiteResult, opts: { baseline?: string; withoutJudgedChanges?: boolean } = {}): string { + // An agent, a caller, a platform or a results file wrote what a run holds: the summary shows a copy that is all text, + // each value on one line, before anything is cut to fit. The comparison is worked out on the run as it is. + const r = printableCopy(result); + const comparison = printableCopy(comparisonOf(result)); const lines: string[] = []; lines.push(bold(`sandtable · ${r.domain}`) + dim(` · agent ${r.agent} · user ${r.user} · k=${r.k}`)); if (r.pendingReview?.length) { @@ -270,7 +275,6 @@ export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withou } } - const comparison = comparisonOf(r); if (comparison) lines.push("", ...renderComparison(comparison, opts.baseline)); if (comparison && !opts.withoutJudgedChanges) lines.push(...spaced(renderJudgedChanges(comparison))); return lines.join("\n"); @@ -278,11 +282,13 @@ export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withou /** * What `sandtable compare` prints: the newer run on two lines, then the block a `run --baseline` summary ends with. - * `r` carries the baseline as `baseline`; `files` names the two results.json files as the command was given them. + * `result` carries the baseline as `baseline`; `files` names the two results.json files as the command was given them. */ -export function renderCompare(r: SuiteResult, files: { baseline: string; results: string }): string { - const comparison = comparisonOf(r); - if (!comparison) throw new Error("renderCompare needs a run with a baseline"); +export function renderCompare(result: SuiteResult, files: { baseline: string; results: string }): string { + const found = comparisonOf(result); + if (!found) throw new Error("renderCompare needs a run with a baseline"); + // As in renderTerminal: a copy that is all text, and the comparison worked out on the run as it is. + const [r, comparison] = [printableCopy(result), printableCopy(found)]; return [ bold(`sandtable compare · ${r.domain} · ${files.results}`), dim(` run ${r.runId} · started ${r.startedAt} · agent ${r.agent} · user ${r.user} · k=${r.k}`), @@ -301,8 +307,8 @@ const spaced = (block: string[]) => (block.length ? ["", ...block] : []); * judged. */ export function renderJudgedChanges(c: Comparison): string[] { - const j = c.judged; - if (!j) return []; + if (!c.judged) return []; + const j = printableCopy(c.judged); const score = (x?: JudgedCounts) => (x === undefined ? "–" : x.score === null ? "n/a" : formatPercent(x.score)); const moved = (b?: JudgedCounts, a?: JudgedCounts) => { if (b === undefined || a === undefined || b.score === null || a.score === null) return ""; diff --git a/src/run/gates.ts b/src/run/gates.ts index 72326073..d302095c 100644 --- a/src/run/gates.ts +++ b/src/run/gates.ts @@ -1,3 +1,4 @@ +import { printable } from "../report/printable.ts"; import { formatPercent } from "../report/terminal.ts"; import { criticalFailures, skippedTrials, tagsBelowFloor, type SuiteResult } from "./suite.ts"; @@ -23,7 +24,7 @@ export function runGates(result: SuiteResult, opts: { failUnder?: number } = {}) const trials = result.tasks.reduce((n, t) => n + t.trials.length, 0); failed.push({ status: 2, - message: `${skipped.length} of ${trials} trial(s) ended in an error before the agent broke any rule, so they were skipped and this run is incomplete: rerun it. The first error:\n ${skipped[0]!.error ?? "unknown error"}`, + message: `${skipped.length} of ${trials} trial(s) ended in an error before the agent broke any rule, so they were skipped and this run is incomplete: rerun it. The first error:\n ${printable(skipped[0]!.error ?? "unknown error")}`, }); } const blockers = criticalFailures(result); diff --git a/src/run/leaderboard.ts b/src/run/leaderboard.ts index 98aa5464..5f39685f 100644 --- a/src/run/leaderboard.ts +++ b/src/run/leaderboard.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { z } from "zod"; import { BRAND_CSS, MARK_SVG } from "../report/brand.ts"; import { renderHtml } from "../report/html.ts"; +import { printableCopy } from "../report/printable.ts"; import type { Domain } from "../spec/schema.ts"; import { noopAgent } from "../sim/noop.ts"; import { scriptedUser } from "../sim/scripted.ts"; @@ -678,7 +679,10 @@ export function softCell(r: LeaderboardRow): string { .join("; "); } -export function renderLeaderboardMarkdown(l: LeaderboardResult): string { +export function renderLeaderboardMarkdown(board: LeaderboardResult): string { + // Labels come from a models file, errors from an agent or its provider, and the rest from the runs: each one line of + // text, with no control character a terminal printing the board would act on. + const l = printableCopy(board); const soft = hasSoft(l); const critical = hasCritical(l); const heads = ["#", "Model", ...passHeads(l.k), ...(critical ? ["Critical"] : []), "Rule violations", "Top failure", ...(soft ? ["Soft rules"] : []), "Cost / trial", "p95 turn"]; diff --git a/src/shell.ts b/src/shell.ts index ae9f86c5..ee5dee35 100644 --- a/src/shell.ts +++ b/src/shell.ts @@ -1,8 +1,21 @@ +import { CONTROL_CHAR } from "./report/printable.ts"; + /** * A path as one shell word, so a printed command runs as it is when the path has spaces, quotes or `$`, or starts * with `-`, which the CLI would read as an option. */ export function shellWord(s: string): string { const p = s.startsWith("-") ? `./${s}` : s; + if (CONTROL_CHAR.test(p)) return ansiQuoted(p); return /^[\w./@:+=,-]+$/.test(p) ? p : `'${p.replaceAll("'", `'\\''`)}'`; } + +/** + * A word with a control character in it, such as a file name from someone else's archive, as `$'…'`: bash and zsh read + * it back as the same text, and the terminal that shows the command sees each control character as an escape, not as one. + */ +function ansiQuoted(s: string): string { + // Each UTF-8 byte as \xHH, which bash 3.2 (macOS's) reads too, where \u needs bash 4.2. + const escape = (c: string) => [...Buffer.from(c, "utf8")].map((b) => `\\x${b.toString(16).padStart(2, "0")}`).join(""); + return `$'${[...s].map((c) => (c === "\\" || c === "'" ? `\\${c}` : CONTROL_CHAR.test(c) ? escape(c) : c)).join("")}'`; +} diff --git a/test/control-characters.test.ts b/test/control-characters.test.ts new file mode 100644 index 00000000..0ab98e90 --- /dev/null +++ b/test/control-characters.test.ts @@ -0,0 +1,354 @@ +import { execFile } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { freightScriptedAgent } from "../src/demo/freight-agents.ts"; +import { calibrate, renderCalibration } from "../src/judge/calibrate.ts"; +import type { JudgedSummary } from "../src/judge/summary.ts"; +import { renderMarkdown } from "../src/report/markdown.ts"; +import { renderCompare, renderJudged, renderJudgedChanges, renderTerminal } from "../src/report/terminal.ts"; +import { baselineSnapshot, comparisonOf } from "../src/run/compare.ts"; +import { renderLeaderboardMarkdown, type LeaderboardRow } from "../src/run/leaderboard.ts"; +import { runSuite, type SuiteResult } from "../src/run/suite.ts"; +import { shellWord } from "../src/shell.ts"; +import { httpAgent } from "../src/sim/http.ts"; +import { openaiCompatAgent } from "../src/sim/openai-compat.ts"; +import { scriptedUser } from "../src/sim/scripted.ts"; +import type { AgentAdapter } from "../src/sim/types.ts"; +import { loadDomain } from "../src/spec/load.ts"; + +// What an agent, a simulated caller, a platform or a results file holds reaches the terminal and Markdown as text: +// no escape sequence, bell or C1 control character in it gets through, and sandtable's own colours still do. + +const example = path.resolve("examples/freight"); +const cli = path.resolve("src/cli.ts"); + +/** + * An OSC that sets the window title and ends with a bell, a CSI that clears the screen, a C1 CSI, a colour, a full reset, + * a cursor save, a switch to line drawing, a C1 OSC ended by a C1 ST, a backspace, a carriage return, DEL and NUL, then + * a right-to-left override and an isolate around text. + */ +const CONTROLS = "\x1b]0;a title\x07\x1b[2J\x9b31m\x1b[5m\x1bc\x1b7\x1b(0\x9d0;t\x9c\x08\r\x7f\x00\u202eevil\u2066x\u2069"; +/** CONTROLS without NUL, which no file name can hold, and without the carriage return, backspace and DEL, so a test can name a file with it. */ +const FILE_NAME_CONTROLS = "\x1b]0;a title\x07\x1b[2J\x9b31m\u202eevil\u2066x\u2069"; +/** CONTROLS as a one-line value shows it, after a space. */ +const SHOWN = "]0;a title [2J 31m [5m c 7 (0 0;t evil x"; +/** Every control character but a newline, and the bidi embedding, override and isolate controls. */ +const STRAY = /[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/g; +/** Each control character left in `s` but a newline, with the text around it. */ +const strays = (s: string) => [...s.matchAll(STRAY)].map((m) => JSON.stringify(s.slice(Math.max(0, m.index - 30), m.index + 10))); + +afterEach(() => vi.unstubAllGlobals()); + +const domain = () => loadDomain(example); +const careful = async (): Promise => runSuite({ domain: await domain(), agent: freightScriptedAgent("careful"), user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + +/** A judged summary of one criterion, as `judge` writes it. */ +const judgedSummary = (id: string, model: string): JudgedSummary => ({ + pass: 0, + fail: 1, + notApplicable: 0, + score: 0, + models: [model], + notJudged: 0, + criteria: [{ id, pass: 0, fail: 1, notApplicable: 0, score: 0, pending: true }], + usage: { [model]: { calls: 1, input: 900, output: 90, cacheRead: 0, cacheWrite: 0 } }, + costUsd: null, +}); + +describe("terminal output strips control characters from run values", () => { + it("shows a tool argument in a failure example as text, also where the line is cut to fit", async () => { + // An OSC whose end a cut at 80 columns would leave out, so the rest of the terminal's output would be its title. + const mc = `X\x1b]0;${"t".repeat(120)}\x07${CONTROLS}`; + const agent: AgentAdapter = { + name: "books", + createSession: async () => ({ + reply: async (_message, callTool) => { + await callTool("book_load", { load_id: "L-1001", mc_number: mc, rate: 2100 }); + return "Booked."; + }, + }), + }; + const r = await runSuite({ domain: await domain(), agent, user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + const out = renderTerminal(r); + expect(out).toContain("e.g. book_load referenced carriers X ]0;tttt"); + expect(strays(out)).toEqual([]); + }); + + it("shows an HTTP agent's error body in skipped trials as text", async () => { + vi.stubGlobal("fetch", async () => new Response(`boom ${CONTROLS}`, { status: 500 })); + const r = await runSuite({ domain: await domain(), agent: httpAgent("http://agent.local/step"), user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + const out = renderTerminal(r); + expect(out).toContain("Skipped trials"); + expect(out).toContain("http://agent.local/step returned 500: boom ]0;a title [2J"); + expect(strays(out)).toEqual([]); + }); + + it("shows an OpenAI-compatible endpoint's error body in skipped trials as text", async () => { + vi.stubGlobal("fetch", async () => new Response(`${CONTROLS} bad request`, { status: 400 })); + const agent = openaiCompatAgent({ baseUrl: "http://llm.local/v1", model: "m" }); + const out = renderTerminal(await runSuite({ domain: await domain(), agent, user: scriptedUser(), taskIds: ["book-at-posted-rate"] })); + expect(out).toContain("http://llm.local/v1/chat/completions returned 400: ]0;a title [2J"); + expect(strays(out)).toEqual([]); + }); + + it("shows the simulated user's sentence in caller notes as text", async () => { + const r = await careful(); + r.tasks[0]!.trials[0]!.callerNotes = [{ turn: 1, why: "talks about its instructions", text: `My instructions say ${CONTROLS}book it.` }]; + const out = renderTerminal(r); + expect(out).toContain(`"My instructions say ${SHOWN} book it."`); + expect(strays(out)).toEqual([]); + }); + + it("shows a results file's agent, task ids, judge models and criteria as text in compare and the judged score", async () => { + const now = await careful(); + now.agent = `agent ${CONTROLS}`; + now.runId = `run ${CONTROLS}`; + now.tasks[0]!.id = `book ${CONTROLS}`; + now.tasks[0]!.passes = 0; + now.tasks[0]!.trials[0]!.judged = { model: `judge ${CONTROLS}`, verdicts: [{ id: `tone ${CONTROLS}`, criterion: `Polite ${CONTROLS}`, verdict: "fail", reason: `Curt ${CONTROLS}`, quote: `No ${CONTROLS}` }] }; + now.judged = judgedSummary(`tone ${CONTROLS}`, `judge ${CONTROLS}`); + const before = structuredClone(now); + before.agent = `old ${CONTROLS}`; + before.user = `caller ${CONTROLS}`; + before.startedAt = `then ${CONTROLS}`; + before.tasks[0]!.passes = 1; + before.tasks[0]!.trials[0]!.judged = { model: `other ${CONTROLS}`, verdicts: [{ id: `tone ${CONTROLS}`, criterion: `Kind ${CONTROLS}`, verdict: "fail", reason: "Curt." }] }; + before.judged = judgedSummary(`tone ${CONTROLS}`, `other ${CONTROLS}`); + const compared: SuiteResult = { ...now, baseline: baselineSnapshot(before) }; + + const outputs = { + compare: renderCompare(compared, { baseline: "baseline.json", results: "results.json" }), + terminal: renderTerminal(compared), + judged: renderJudged(now).join("\n"), + judgedChanges: renderJudgedChanges(comparisonOf(compared)!).join("\n"), + }; + expect(outputs.compare).toContain("broken"); + expect(outputs.compare).toContain(`agent old ${SHOWN} → agent ${SHOWN}`); + expect(outputs.judged).toContain(`by judge ${SHOWN}`); + expect(outputs.judgedChanges).toContain("text differs"); + for (const [name, out] of Object.entries(outputs)) expect(strays(out), name).toEqual([]); + }); + + it("shows a results file's task, criterion, model, quote and reason as text in a calibration", () => { + const [task, criterion, model] = [`a ${CONTROLS}`, `tone ${CONTROLS}`, `judge ${CONTROLS}`]; + const run = { + version: 1, + domain: "freight", + agent: "scripted:careful", + passHat: {}, + tasks: [{ id: task, trials: [{ taskId: task, trial: 0, judged: { model, verdicts: [{ id: criterion, criterion: "Polite.", verdict: "fail", quote: `No ${CONTROLS}`, reason: `Curt ${CONTROLS}` }] } }] }], + judged: judgedSummary(criterion, model), + } as unknown as SuiteResult; + const out = renderCalibration(calibrate(run, [{ task, trial: 1, criterion, label: "pass" }])); + expect(out).toContain(`Judge judge ${SHOWN}`); + expect(out).toContain(`quote: "No ${SHOWN}"`); + expect(strays(out)).toEqual([]); + }); + + it("keeps sandtable's own colours on a terminal, and only those", async () => { + const tty = process.stdout.isTTY; + const noColor = process.env.NO_COLOR; + vi.resetModules(); + process.stdout.isTTY = true; + delete process.env.NO_COLOR; + try { + const terminal = await import("../src/report/terminal.ts"); + const r = await careful(); + r.agent = `agent ${CONTROLS}`; + r.tasks[0]!.passes = 0; + r.tasks[0]!.trials[0]!.callerNotes = [{ turn: 1, why: "talks about its instructions", text: `My instructions say ${CONTROLS}` }]; + const out = terminal.renderTerminal(r); + // Red, green, yellow, bold and dim, each reset after. + expect(out).toContain("\x1b[31m0/1\x1b[0m"); + expect(strays(out.replace(/\x1b\[(?:0|1|2|31|32|33)m/g, ""))).toEqual([]); + } finally { + process.stdout.isTTY = tty; + if (noColor !== undefined) process.env.NO_COLOR = noColor; + vi.resetModules(); + } + }); +}); + +describe("Markdown output strips control characters from run values", () => { + it("report --markdown shows a failure example, the agent, ids and a verdict's reason as text", async () => { + const r = await careful(); + r.agent = `agent ${CONTROLS}`; + r.tasks[0]!.passes = 0; + r.tasks[0]!.trials[0]!.judged = { model: "claude-sonnet-5-5", verdicts: [{ id: "tone", criterion: "Polite.", verdict: "fail", reason: `Curt ${CONTROLS}`, quote: `No ${CONTROLS}` }] }; + r.judged = judgedSummary("tone", "claude-sonnet-5-5"); + r.failureModes = [{ kind: "invariant", id: `rule ${CONTROLS}`, count: 1, tasks: [r.tasks[0]!.id], example: `book_load referenced carriers X${CONTROLS}` }]; + const md = renderMarkdown(r); + expect(md).toContain("Top failure modes"); + expect(md).toContain("Curt \\]0;a title \\[2J 31m \\[5m c 7 \\(0 0;t evil x"); + expect(strays(md)).toEqual([]); + }); + + it("LEADERBOARD.md shows the domain, a model's label, its failure, rule and task ids and an error as text", () => { + const row: LeaderboardRow = { + label: `model ${CONTROLS}`, + agent: "scripted:careful", + baseline: false, + passHat1: 0, + passHatK: 0, + tasks: 1, + trials: 1, + violations: { [`rule ${CONTROLS}`]: 1 }, + topFailure: `rule ${CONTROLS} (1×)`, + soft: { [`soft ${CONTROLS}`]: { applied: 1, broken: 1, rate: 1 } }, + critical: { tasks: 1, failed: [`task ${CONTROLS}`] }, + costUsd: null, + costPerTrialUsd: null, + agentCostPerTrialUsd: null, + agentTurnP95Ms: null, + agentTokens: 0, + }; + const md = renderLeaderboardMarkdown({ + version: 1, + domain: `lanes ${CONTROLS}`, + user: `caller ${CONTROLS}`, + k: 1, + generatedAt: `now ${CONTROLS}`, + sandtable: { version: `1.5.0 ${CONTROLS}` }, + rows: [row, { ...row, label: "crashed", error: `boom ${CONTROLS}` }], + }); + expect(md).toContain(`# Leaderboard: lanes ${SHOWN}`); + expect(md).toContain(`boom ${SHOWN}`); + expect(strays(md)).toEqual([]); + }); +}); + +describe.skipIf(!process.features.typescript)("CLI lines strip control characters from run values", () => { + // No key or AWS setting: a model call, if one slipped through, fails instead of costing money. + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("AWS_") && !["ANTHROPIC_API_KEY", "OPENAI_API_KEY"].includes(k))); + const sandtable = (cwd: string, ...args: string[]) => + promisify(execFile)(process.execPath, [cli, ...args], { env, cwd }).then( + (r) => ({ code: 0, ...r }), + (e: { code: number; stdout: string; stderr: string }) => e, + ); + + it("prints an agent's error as text: every trial ended in one, a skipped trial's gate, and a leaderboard model that failed", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const error = JSON.stringify(`no credentials ${CONTROLS}`); + await writeFile(path.join(dir, "crash.mjs"), `export default { name: "crash", createSession: async () => ({ reply: async () => { throw new Error(${error}); } }) };\n`); + await writeFile( + path.join(dir, "some.mjs"), + `export default { name: "some", createSession: async (ctx) => ({ reply: async () => { if (ctx.sessionId.includes("walk-away")) throw new Error(${error}); return "Sorry, I can't help with that."; } }) };\n`, + ); + await writeFile(path.join(dir, "models.yaml"), "models:\n - { label: crash, agent: ./crash.mjs }\n"); + const run = (agent: string, tasks: string) => sandtable(dir, "run", example, "--agent", agent, "--user", "scripted", "-t", tasks, "--no-html", "-o", path.join(dir, `out-${path.basename(agent, ".mjs")}`)); + const all = await run("./crash.mjs", "book-at-posted-rate"); + expect(all.stderr).toContain("Every trial ended in an error"); + expect(all.stderr).toContain(`no credentials ${SHOWN}`); + const some = await run("./some.mjs", "expired-insurance,walk-away-above-ceiling"); + expect(some.stderr).toContain("ended in an error before the agent broke any rule"); + const board = await sandtable(dir, "leaderboard", example, "-m", "models.yaml", "--user", "scripted", "-k", "1", "-t", "book-at-posted-rate", "-o", path.join(dir, "board")); + expect(board.stdout).toContain("✗ failed"); + expect(board.stdout).toContain(`no credentials ${SHOWN}`); + for (const r of [all, some, board]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("prints a results file's values as text: compare's summary and regression gate, calibrate's header, and errors that name the run's domain", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const now = await careful(); + now.domain = `freight ${CONTROLS}`; + now.agent = `agent ${CONTROLS}`; + now.tasks[0]!.trials[0]!.judged = { model: "claude-sonnet-5-5", verdicts: [{ id: "tone", criterion: "Polite.", verdict: "fail", reason: "Curt." }] }; + now.judged = judgedSummary("tone", "claude-sonnet-5-5"); + const failing = { ...now, tasks: [{ ...now.tasks[0]!, id: `book ${CONTROLS}`, passes: 0 }] }; + const passing = { ...failing, tasks: [{ ...failing.tasks[0]!, passes: 1 }] }; + const other = { ...now, domain: `clinic ${CONTROLS}` }; + for (const [file, r] of Object.entries({ "now.json": now, "failing.json": failing, "passing.json": passing, "other.json": other })) await writeFile(path.join(dir, file), JSON.stringify(r)); + await writeFile(path.join(dir, "labels.json"), JSON.stringify({ labels: [{ task: now.tasks[0]!.id, trial: 1, criterion: "tone", label: "pass" }] })); + const compare = await sandtable(dir, "compare", "passing.json", "failing.json", "--fail-on-regression"); + expect(compare.code).toBe(1); + expect(compare.stdout).toContain(`sandtable compare · freight ${SHOWN}`); + expect(compare.stderr).toContain(`broke since the baseline (--fail-on-regression): book ${SHOWN} 1/1 → 0/1`); + const header = await sandtable(dir, "calibrate", "now.json", "--labels", "labels.json"); + expect(header.stdout).toContain(`sandtable calibrate · freight ${SHOWN} · agent agent ${SHOWN}`); + const errors = [ + await sandtable(dir, "compare", "now.json", "other.json"), + await sandtable(dir, "regrade", "now.json", "-d", example, "-o", path.join(dir, "regraded")), + await sandtable(dir, "judge", "now.json", "-d", example, "--dry-run"), + ]; + expect(errors[0]!.stderr).toContain("now.json and other.json are runs of different domains, freight "); + expect(errors[1]!.stderr).toContain("can't regrade now.json: it is a run of freight "); + expect(errors[2]!.stderr).toContain("can't judge now.json: it is a run of freight "); + for (const r of errors) expect(r.code).toBe(2); + for (const r of [compare, header, ...errors]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("review prints a drafted item's quote as text, keeping its lines", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const copy = path.join(dir, "freight"); + await cp(example, copy, { recursive: true }); + const file = path.join(copy, "sandtable.yaml"); + const yaml = (await readFile(file, "utf8")) + .replace(' source: { file: policy.md, quote: "Ask for', ' review: pending\n source: { file: policy.md, quote: "\\e]0;a title\\a\\e[2J\\x9b31m\\ec\\e7\\e(0\\x9d0;t\\x9c\\b\\r\\x7f\\u202eevil\\u2066\\NAsk for') + .replace(' source: { file: policy.md, quote: "Do **not** book', ' review: pending\n source: { file: policy.md, quote: "Do **not** book'); + await writeFile(file, yaml); + const r = await sandtable(dir, "review", copy); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout).toContain(' " ]0;a title [2J 31m c 7 (0 0;t evil Ask for the carrier'); + expect(r.stdout).toContain('"Do **not** book a carrier if any of these is true:\n - authority type is not `carrier`'); + expect(strays(r.stdout)).toEqual([]); + }); + + it("validate, lint and review print a drafted domain's name and task ids as text", async () => { + // `init` copies a model's ids into tasks.yaml, where YAML keeps an escape as `\e`: it loads back as one. + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const copy = path.join(dir, "freight"); + await cp(example, copy, { recursive: true }); + const spec = path.join(copy, "sandtable.yaml"); + await writeFile(spec, (await readFile(spec, "utf8")).replace("name: freight-carrier-sales", 'name: "freight\\e]0;a title\\a\\x9b2J\\u202eevil"')); + const tasks = path.join(copy, "tasks.yaml"); + await writeFile(tasks, (await readFile(tasks, "utf8")).replace("- id: book-at-posted-rate\n", '- id: "book\\e]0;a title\\a\\x9b2Jat-rate"\n review: pending\n')); + const validate = await sandtable(dir, "validate", copy); + expect(validate.code, validate.stderr).toBe(0); + expect(validate.stdout).toContain("✓ freight ]0;a title 2J evil: "); + const lint = await sandtable(dir, "lint", copy); + expect(lint.stdout).toContain("book ]0;a title 2Jat-rate (write"); + const review = await sandtable(dir, "review", copy); + expect(review.code, review.stderr).toBe(0); + expect(review.stdout).toContain("- task book ]0;a title 2Jat-rate"); + for (const r of [validate, lint, review]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("init --transcripts prints a transcript's file name as text", async () => { + // A folder of chats from someone else's archive: its file names are as outside as the chats. + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const chats = path.join(dir, "chats"); + await mkdir(chats); + await cp(example, path.join(dir, "draft"), { recursive: true }); + await writeFile(path.join(chats, `call${FILE_NAME_CONTROLS}.txt`), "Caller: I'd like to book load L-1008 for 2026-03-16.\nAgent: Let me check that load.\n"); + const r = await sandtable(dir, "init", "--transcripts", "chats", "--call-date", "2026-03-14", "-o", "draft", "--dry-run"); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout).toContain(" call ]0;a title [2J 31m evil x .txt: 2026-03-16, stated in the file"); + expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("a printed command quotes a word with control characters so the shell reads it back as it was", async () => { + const word = `chats/call${FILE_NAME_CONTROLS}'s.txt`; + const quoted = shellWord(word); + expect(strays(quoted)).toEqual([]); + const { stdout } = await promisify(execFile)("bash", ["-c", `printf %s ${quoted}`]); + expect(stdout).toBe(word); + }); + + it("init --openapi prints a spec's refs as text in its warnings", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const schema = { $ref: `./missing${CONTROLS}.yaml` }; + const spec = { + openapi: "3.0.0", + info: { title: "Controls", version: "1" }, + paths: { "/loads": { post: { operationId: "book_load", requestBody: { content: { "application/json": { schema } } }, responses: { "200": { description: "ok" } } } } }, + }; + await writeFile(path.join(dir, "api.json"), JSON.stringify(spec)); + const r = await sandtable(dir, "init", "--openapi", "api.json", "-o", "draft"); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout + r.stderr).toContain("missing ]0;a title [2J 31m"); + expect(strays(r.stdout + r.stderr)).toEqual([]); + }); +}); diff --git a/test/judge.test.ts b/test/judge.test.ts index 02044582..b5b2eedc 100644 --- a/test/judge.test.ts +++ b/test/judge.test.ts @@ -549,4 +549,14 @@ describe.skipIf(!process.features.typescript)("the judge command", () => { expect(r.stderr).toContain("2 trial(s) were not judged"); expect(await readFile(f.results, "utf8")).toBe(f.text); }); + + it("prints a failed call's error as text, whether the call failed alone or stopped the judge", { timeout: 30_000 }, async () => { + const f = await fixture(); + const controls = "\x1b]0;a title\x07\x1b[2J\x9b31m\x1bc\x08\r\x7f\u202eevil\u2069"; + const alone = await judgeThrough((await stub(() => ({ status: 400, body: { message: `prompt is too long ${controls}` } }))).base, f.results, "-d", f.domainDir, "-c", "1", "--no-html"); + expect(alone.stdout).toContain("the judge call failed: 400 prompt is too long ]0;a title [2J 31m c evil"); + const stopped = await judgeThrough((await stub(() => ({ status: 401, body: { message: `invalid x-api-key ${controls}` } }))).base, f.results, "-d", f.domainDir, "-c", "1", "--no-html"); + expect(stopped.stderr).toMatch(/judge stopped.*invalid x-api-key/); + for (const r of [alone, stopped]) expect(r.stdout + r.stderr).not.toMatch(/[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/); + }); }); diff --git a/test/serve-driven.test.ts b/test/serve-driven.test.ts index 283382c4..127d0e16 100644 --- a/test/serve-driven.test.ts +++ b/test/serve-driven.test.ts @@ -500,6 +500,33 @@ describe("the serve command", () => { expect((await readFile(path.join(dir, "report.html"), "utf8")).length).toBeGreaterThan(1000); }); + it.skipIf(!process.features.typescript)("prints a platform's call id as text, in its live log and its summary", { timeout: 30_000 }, async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-serve-cli-")); + const port = await freePort(); + const child = spawn(process.execPath, [cli, "serve", example, "--task", "book-at-posted-rate", "--port", String(port), "-o", dir, "--no-html"], { stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + child.stdout.on("data", (d) => (out += d)); + child.stderr.on("data", (d) => (out += d)); + const url = `http://127.0.0.1:${port}`; + for (let i = 0; i < 100 && !out.includes("Results:"); i++) await new Promise((r) => setTimeout(r, 100)); + // Requests with no call id, and a transcript that lists other calls: the trial's error names the call. + await json(`${url}/carriers/100101`); + await json(`${url}/carriers/100101`); + await post(`${url}/bookings`, BOOK); + const callId = "conv-\x1b]0;a title\x07\x1b[2J\x9b31m\x1bc\x9d0;t\x9c\u202eevil"; + const tools = ["verify_carrier", "book_load"].map((tool_name) => ({ tool_name, params_as_json: "{}", tool_has_been_called: true, type: "webhook" })); + const end = { type: "post_call_transcription", event_timestamp: 1, data: { conversation_id: callId, transcript: [{ role: "agent", message: null, tool_calls: tools }, { role: "agent", message: "L-1001 is booked.", tool_calls: null }] } }; + expect((await post(`${url}/webhook`, end)).body).toMatchObject({ skipped: true }); + const code = await new Promise((resolve) => { + child.on("exit", resolve); + child.kill("SIGINT"); + }); + expect(code).toBe(0); + expect(out).toContain("○ book-at-posted-rate trial 1: runtime (call conv- ]0;a title [2J 31m c 0;t evil's transcript lists its tool calls as verify_carrier, book_load"); + expect(out).toContain("Skipped trials"); + expect(out).not.toMatch(/[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/); + }); + it.skipIf(!process.features.typescript)("refuses an --idle-timeout that isn't a whole number of minutes", { timeout: 30_000 }, async () => { const results = await Promise.all( ["-1", "abc", "1.5"].map((v) => promisify(execFile)(process.execPath, [cli, "serve", example, "--idle-timeout", v]).catch((err: { code: number; stderr: string }) => err)),