diff --git a/CHANGELOG.md b/CHANGELOG.md index c91b7a5..8514010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/2.0.0/), ## [Unreleased] +### Security + +- Terminal output, `report --markdown` and `LEADERBOARD.md`: control characters, and the bidi controls that show text reversed, now show as spaces in every value that comes from an agent, a simulated caller, a platform's call, a results file, a transcript's file name or a domain that `init` drafted, and a command sandtable suggests quotes such a file name so the shell still reads it right. Before, such text could write escape sequences to the terminal of whoever ran `run`, `serve`, `compare`, `regrade`, `leaderboard`, `review`, `judge` or `calibrate`, and set the window title, clear the screen or hide the output that follows. Upgrade if you run `serve` behind a tunnel or read results files you didn't write ([GHSA-qrx5-3qq7-7w59](https://github.com/sandtable-ai/sandtable/security/advisories/GHSA-qrx5-3qq7-7w59)). + ## [1.5.0] - 2026-11-09 A minor release with no breaking changes, and a security fix in the HTML reports: upgrade if you open reports built from domains or results files you didn't write. `sandtable judge` and `run --judge` score a run on judged criteria with a model, apart from pass^k, which they never change, and `sandtable calibrate` measures a judge against human labels. Clock dates reach the policy, param defaults, fault messages and `reply_includes`, and `report --markdown` writes a summary for a CI job or a pull request comment. diff --git a/src/cli.ts b/src/cli.ts index 6c6d5d0..ea9f53d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { draftUsageLines, formatPercent, renderCompare, renderJudged, renderJudg import { renderHtml } from "./report/html.ts"; import { renderCustomerHtml } from "./report/customer.ts"; import { renderMarkdown } from "./report/markdown.ts"; +import { printable, printableLines } from "./report/printable.ts"; import { everyTrialFailed, runSuite, trialStatus, type SuiteResult } from "./run/suite.ts"; import { runGates } from "./run/gates.ts"; import { startWorldServer } from "./serve/server.ts"; @@ -253,7 +254,7 @@ function selectTasks(domain: Domain, taskIds: string[] | undefined, tags: string function failIfNothingGraded(result: SuiteResult): boolean { const error = everyTrialFailed(result); if (error === undefined) return false; - console.error(`\n✗ Every trial ended in an error, so this run says nothing about the agent. The first error:\n ${error}`); + console.error(`\n✗ Every trial ended in an error, so this run says nothing about the agent. The first error:\n ${printable(error)}`); exitWith(EXIT_ERROR); return true; } @@ -274,7 +275,7 @@ function failIfRegressed(comparison: Comparison): void { const broken = regressions(comparison); if (broken.length === 0) return; const counts = (x?: { passes: number; trials: number }) => (x ? `${x.passes}/${x.trials}` : "-"); - console.error(`\n✗ ${broken.length} task(s) broke since the baseline (--fail-on-regression): ${broken.map((d) => `${d.id} ${counts(d.before)} → ${counts(d.after)}`).join(", ")}`); + console.error(`\n✗ ${broken.length} task(s) broke since the baseline (--fail-on-regression): ${broken.map((d) => `${printable(d.id)} ${counts(d.before)} → ${counts(d.after)}`).join(", ")}`); exitWith(EXIT_GATE); } @@ -287,7 +288,7 @@ async function loadForCheck(domainPath: string): Promise { return await loadDomain(await resolveDomain(domainPath)); } catch (err) { if (!(err instanceof DomainError) || err instanceof DomainNotFoundError) throw err; - console.error(`Invalid domain: ${err.message}`); + console.error(printableLines(`Invalid domain: ${err.message}`)); exitWith(EXIT_GATE); return undefined; } @@ -383,7 +384,7 @@ function callDateLines(transcripts: { file: string; text: string }[], fallback: "Call dates, which each chat's drafted dates are made relative to:", ...transcripts.map((t) => { const stated = callDateIn(t.text); - return ` ${t.file}: ${stated !== undefined ? `${stated}, stated in the file` : fallback !== undefined ? `${fallback}, from --call-date` : "none: its dates stay literal"}`; + return ` ${printable(t.file)}: ${stated !== undefined ? `${stated}, stated in the file` : fallback !== undefined ? `${fallback}, from --call-date` : "none: its dates stay literal"}`; }), ]; } @@ -493,13 +494,13 @@ withProviderFlags(program openapiDraft = { draft, files }; if (o.dryRun) { draftedDomainChars = JSON.stringify(YAML.parse(files[DOMAIN_FILE]!)).length + files["policy.md"].length; - console.log(`Would draft ${draft.name} from ${o.openapi}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools. No model call.`); + console.log(`Would draft ${printable(draft.name)} from ${o.openapi}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools. No model call.`); for (const [f, text] of Object.entries(files)) { const target = path.join(o.out, f); const exists = await stat(target).then(() => true, () => false); console.log(` would write ${target}${exists && !o.force ? " (exists: init stops unless you pass --force)" : ""} (${text.length} characters)`); } - for (const w of draft.warnings) console.log(` ! ${w}`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); } } if (o.dryRun) { @@ -518,8 +519,8 @@ withProviderFlags(program const { draft, files } = openapiDraft; const written = await writeDraft(o.out, files, o.force); const notes = draft.tools.reduce((n, t) => n + t.notes.length, 0); - console.log(`Drafted ${draft.name}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools, ${notes} review notes.`); - for (const w of draft.warnings) console.log(` ! ${w}`); + console.log(`Drafted ${printable(draft.name)}: ${Object.keys(draft.tables).length} tables, ${draft.tools.length} tools, ${notes} review notes.`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); for (const f of written) console.log(` wrote ${f}`); } if (sopText !== undefined || transcripts) { @@ -544,7 +545,7 @@ withProviderFlags(program const file = await appendToDomainList(o.out, "invariants", res.invariants, softRuleComments(res.softRules)); console.log(` ${res.invariants.length} invariant(s) drafted (review: pending), ${res.softRules.length} soft rule(s) noted in ${file}`); for (const l of draftUsageLines(res.usage)) console.log(l); - for (const w of res.warnings) console.log(` ! ${w}`); + for (const w of res.warnings) console.log(` ! ${printable(w)}`); } // Transcripts that drafted nothing: those whose call failed, and those never sent once a failure stopped drafting; // `all` when that is every transcript. @@ -571,7 +572,7 @@ withProviderFlags(program } if (res.literal.length) console.log(` ! ${res.literal.length} task(s) keep literal dates, since their chat states no call date: pass --call-date, or see their TODO(review) notes`); for (const l of draftUsageLines(res.usage)) console.log(l); - for (const w of res.warnings) console.log(` ! ${w}`); + for (const w of res.warnings) console.log(` ! ${printable(w)}`); if (res.failed.length) undrafted = { failed: res.failed, notSent: res.notSent, all: res.failed.length + res.notSent.length === transcripts.length }; } // Nothing to review when the transcripts were the only step and none of them drafted. @@ -588,7 +589,7 @@ withProviderFlags(program if (notSent.length) { // An error every later call would hit too stopped drafting: it is named once, at the chat it hit. const sent = notSent.length === 1 ? "1 transcript wasn't sent" : `${notSent.length} transcripts weren't sent`; - console.error(`\n✗ Drafting stopped at ${failed.at(-1)!.file}: its call failed in a way every later call would too, so ${sent}. ${all ? "Nothing was drafted from the transcripts:" : others}`); + console.error(`\n✗ Drafting stopped at ${printable(failed.at(-1)!.file)}: its call failed in a way every later call would too, so ${sent}. ${all ? "Nothing was drafted from the transcripts:" : others}`); } else { console.error( all @@ -596,8 +597,8 @@ withProviderFlags(program : `\n✗ ${failed.length} of ${n} transcripts' drafting calls failed, so nothing was drafted from ${failed.length === 1 ? "it" : "them"}. ${others}`, ); } - for (const f of failed) console.error(` ${f.file}: ${f.error}`); - if (notSent.length) console.error(` not sent: ${notSent.join(", ")}`); + for (const f of failed) console.error(` ${printable(f.file)}: ${printable(f.error)}`); + if (notSent.length) console.error(` not sent: ${notSent.map(printable).join(", ")}`); const paths = all ? [o.transcripts] : [...failed.map((f) => f.file), ...notSent].map((file) => transcripts.find((t) => t.file === file)!.path); console.error(` Draft ${left === 1 ? "it" : "them"} again with:`); for (const p of paths) console.error(` ${transcriptsRerun(p, o)}`); @@ -662,8 +663,8 @@ program const rendered = importer.render(draft); const written = await writeDraft(o.out, { [DOMAIN_FILE]: rendered.domain, "policy.md": draft.policy, "tasks.yaml": rendered.tasks }, o.force); const criteria = draft.tasks.reduce((n, t) => n + (t.judged?.length ?? 0), 0); - console.log(`Imported ${draft.name} from ${importer.label}: ${draft.tools.length} tools, ${Object.keys(draft.tables).length} tables, ${draft.tasks.length} tasks, ${criteria} judged ${criteria === 1 ? "criterion" : "criteria"}, all pending review.`); - for (const w of draft.warnings) console.log(` ! ${w}`); + console.log(`Imported ${printable(draft.name)} from ${importer.label}: ${draft.tools.length} tools, ${Object.keys(draft.tables).length} tables, ${draft.tasks.length} tasks, ${criteria} judged ${criteria === 1 ? "criterion" : "criteria"}, all pending review.`); + for (const w of draft.warnings) console.log(` ! ${printable(w)}`); for (const f of written) console.log(` wrote ${f}`); console.log( `\nNext: accept or fix each drafted item, draft the rules from the prompt, and validate:\n ${NEXT} review ${shellWord(o.out)}\n ${NEXT} init --sop ${shellWord(path.join(o.out, "policy.md"))} -o ${shellWord(o.out)} --dry-run\n ${NEXT} validate ${shellWord(o.out)} --strict\n${importer.serve(o.out)}`, @@ -678,8 +679,8 @@ program .action(async (domainPath: string, o: { strict?: boolean }) => { const d = await loadForCheck(domainPath); if (!d) return; - console.log(`✓ ${d.name}: ${Object.keys(d.world.tables).length} tables, ${d.tools.length} tools, ${d.invariants.length} invariants, ${d.tasks.length} tasks`); - for (const w of domainWarnings(d)) console.log(`⚠ ${w}`); + console.log(`✓ ${printable(d.name)}: ${Object.keys(d.world.tables).length} tables, ${d.tools.length} tools, ${d.invariants.length} invariants, ${d.tasks.length} tasks`); + for (const w of domainWarnings(d)) console.log(`⚠ ${printableLines(w)}`); const pending = pendingReview(d); if (pending.length) { console.log(`⚠ ${pending.length} drafted item(s) pending review (${NEXT} review ${shellWord(domainPath)})`); @@ -699,15 +700,15 @@ program const solvable = t.solvable === null ? "solvable ?" : t.solvable ? "solvable" : "NOT solvable"; const kind = t.expectsWrite ? "write" : "no-write"; const mark = t.errors.length ? "✗" : t.warnings.length ? "!" : "✓"; - console.log(`${mark} ${t.id} (${kind} · ${t.ready ? "ready" : "NOT ready"} · ${solvable} · noop ${t.noopPasses ? "passes" : "fails"})`); - for (const e of t.errors) console.log(` ✗ ${e}`); - for (const w of t.warnings) console.log(` ! ${w}`); + console.log(`${mark} ${printable(t.id)} (${kind} · ${t.ready ? "ready" : "NOT ready"} · ${solvable} · noop ${t.noopPasses ? "passes" : "fails"})`); + for (const e of t.errors) console.log(` ✗ ${printableLines(e)}`); + for (const w of t.warnings) console.log(` ! ${printableLines(w)}`); } // Rules no task puts in play: a green run says nothing about them. A tool no task uses is a note, not a warning. const idle = report.rules.filter((r) => r.warnings.length); if (report.rules.length) console.log(`\n${report.rules.length - idle.length} of ${report.rules.length} rules in play in at least one task.`); - for (const r of idle) for (const w of r.warnings) console.log(`! ${r.id}: ${w}`); - if (report.unusedTools.length) console.log(`Tools no task uses: ${report.unusedTools.join(", ")}`); + for (const r of idle) for (const w of r.warnings) console.log(`! ${printable(r.id)}: ${printableLines(w)}`); + if (report.unusedTools.length) console.log(`Tools no task uses: ${report.unusedTools.map(printable).join(", ")}`); const noWrite = report.tasks.filter((t) => !t.expectsWrite).length; const noopPasses = report.tasks.filter((t) => t.noopPasses).length; console.log(`\n${report.tasks.length} tasks: ${report.tasks.length - noWrite} expect a write, ${noWrite} do not; a do-nothing agent passes ${noopPasses} of ${report.tasks.length}.`); @@ -726,9 +727,10 @@ program // The notes `init` left above each item, such as a status code to encode or a value copied from a chat. const notes = await reviewNotes(target); for (const p of pending) { - console.log(`- ${p.kind} ${p.id}${p.source ? ` (${p.source.file})` : ""}`); - if (p.source?.quote) console.log(` "${p.source.quote}"`); - for (const n of notes.get(`${p.kind} ${p.id}`) ?? []) console.log(` TODO(review): ${n}`); + console.log(`- ${p.kind} ${printable(p.id)}${p.source ? ` (${printable(p.source.file)})` : ""}`); + // A quote can be a caller's words from a transcript, and a note a model's: text, never escape sequences. + if (p.source?.quote) console.log(` "${printableLines(p.source.quote)}"`); + for (const n of notes.get(`${p.kind} ${p.id}`) ?? []) console.log(` TODO(review): ${printable(n)}`); } console.log(`\nAccept an item by changing its \`review: pending\` to \`review: accepted\` (or deleting the line).`); }); @@ -756,11 +758,11 @@ function judgeCommand(results: string, domainPath: string, o: RunJudgeFlags): st */ async function judgeFinishedRun(result: SuiteResult, domain: Domain, domainPath: string, out: string, o: RunJudgeFlags): Promise { if (!statesCriteria(domain, result)) { - console.log(`\n${domain.name} states no judged criteria for the tasks of this run, so there is nothing to judge.`); + console.log(`\n${printable(domain.name)} states no judged criteria for the tasks of this run, so there is nothing to judge.`); return; } const failure = (what: string) => - console.error(`\n✗ ${what}\n The run's results are saved, and its gates alone decide the exit code. To judge it: ${judgeCommand(path.join(out, "results.json"), domainPath, o)}`); + console.error(`\n✗ ${printableLines(what)}\n The run's results are saved, and its gates alone decide the exit code. To judge it: ${judgeCommand(path.join(out, "results.json"), domainPath, o)}`); try { const claude = { ...providerOpts(o), model: o.judgeModel ?? DEFAULT_JUDGE_MODEL }; const plan = judgePlan(domain, result, claude); @@ -777,14 +779,14 @@ async function judgeFinishedRun(result: SuiteResult, domain: Domain, domainPath: claude, concurrency: o.concurrency, onTrial: ({ trial, result: judged, warnings }) => { - const label = `${trial.taskId} trial ${trial.trial + 1}`; + const label = `${printable(trial.taskId)} trial ${trial.trial + 1}`; const tally = (verdict: string, word: string) => { const n = trial.judged!.verdicts.filter((v) => v.verdict === verdict).length; return n ? [`${n} ${word}`] : []; }; const verdicts = [...tally("pass", "pass"), ...tally("fail", "fail"), ...tally("not_applicable", "not applicable")].join(", "); - console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${trial.judged!.error}` : ` ${label}: ${verdicts || "no verdict"}`); - for (const w of warnings) console.log(` ! ${w}`); + console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${printable(trial.judged!.error)}` : ` ${label}: ${verdicts || "no verdict"}`); + for (const w of warnings) console.log(` ! ${printable(w)}`); writing = writing.then(() => save(judged, out, false).then(() => undefined)); return writing; }, @@ -883,7 +885,7 @@ withProviderFlags(program ...(server ? { server } : {}), onTrial: (r) => { done++; - if (process.stderr.isTTY) process.stderr.write(`\r ${done}/${total} trials ${STATUS_MARK[trialStatus(r)]} ${r.taskId}`.padEnd(70)); + if (process.stderr.isTTY) process.stderr.write(`\r ${done}/${total} trials ${STATUS_MARK[trialStatus(r)]} ${printable(r.taskId)}`.padEnd(70)); }, }).finally(() => server?.stop()); if (process.stderr.isTTY) process.stderr.write("\r".padEnd(72) + "\r"); @@ -939,11 +941,11 @@ program label: o.label, idleTimeout: o.idleTimeout, ...(task ? { task } : {}), - onOpen: (s) => console.log(` + ${s.id} opened: a trial of ${s.task.id}`), + onOpen: (s) => console.log(` + ${s.id} opened: a trial of ${printable(s.task.id)}`), onGraded: (trial, result) => { const mark = STATUS_MARK[trialStatus(trial)]; - const failed = trial.checks.filter((c) => !c.passed).map((c) => c.id); - console.log(` ${mark} ${trial.taskId} trial ${trial.trial + 1}${failed.length ? `: ${failed.join(", ")}` : ""}${trial.error ? ` (${trial.error})` : ""}`); + const failed = trial.checks.filter((c) => !c.passed).map((c) => printable(c.id)); + console.log(` ${mark} ${printable(trial.taskId)} trial ${trial.trial + 1}${failed.length ? `: ${failed.join(", ")}` : ""}${trial.error ? ` (${printable(trial.error)})` : ""}`); // One write at a time, in order, so the files always hold the latest grade; the end is answered after it. writing = writing.then(() => save(result, out, o.html).then(() => undefined)).catch((err: unknown) => console.error(` ! could not write ${out}: ${err instanceof Error ? err.message : String(err)}`)); return writing; @@ -954,8 +956,8 @@ program // Before it says it's serving, so no conversation is graded with nowhere to write it. await prepareOutFolder(out, outFolder("serve", o.out)); const url = served.server.url; - console.log(`Serving ${domain.name} at ${url} for conversations another system drives, until you stop it (Ctrl-C).`); - console.log(` Each conversation is a trial of ${task ? task.id : "the task it names (POST /sessions with { \"task\": \"\" })"}: REST ${url}/tools/, MCP ${url}/mcp, webhooks ${url}/webhook.`); + console.log(`Serving ${printable(domain.name)} at ${url} for conversations another system drives, until you stop it (Ctrl-C).`); + console.log(` Each conversation is a trial of ${task ? printable(task.id) : "the task it names (POST /sessions with { \"task\": \"\" })"}: REST ${url}/tools/, MCP ${url}/mcp, webhooks ${url}/webhook.`); console.log(` A conversation ends with POST ${url}/sessions//end, a Vapi end-of-call report, Retell's call_ended or an MCP DELETE, and is graded then.`); if (o.idleTimeout > 0) { console.log(` One with no request for ${o.idleTimeout} minute${o.idleTimeout === 1 ? "" : "s"} is ended without its end, and skipped unless the agent broke a rule (--idle-timeout).`); @@ -1025,7 +1027,7 @@ withProviderFlags(program ...(uses.claude ? { provider: providerOf(p) } : {}), ...(region ? { region } : {}), }; - console.log(`Leaderboard: ${cfg.models.length} model(s) on ${domain.name} (${selected.length} tasks, k=${k}, caller ${user.name})\n`); + console.log(`Leaderboard: ${cfg.models.length} model(s) on ${printable(domain.name)} (${selected.length} tasks, k=${k}, caller ${user.name})\n`); if (runInfo.sandtable.dirty) console.log(" ! the sandtable checkout has uncommitted changes: run.json records the commit, not them\n"); const mark = { cached: "↺ cached ", regraded: "↻ regraded", ran: "✓ ran ", failed: "✗ failed ", skipped: "- skipped " } as const; const result = await runLeaderboard({ @@ -1041,7 +1043,7 @@ withProviderFlags(program userSpec, ...(prices ? { prices } : {}), resolveAgent: (spec, effort) => leaderboardAgent(spec, effort, p, o.openaiBaseUrl), - onEntry: (label, status, note) => console.log(` ${mark[status]} ${label}${note ? ` (${note})` : ""}`), + onEntry: (label, status, note) => console.log(` ${mark[status]} ${printable(label)}${note ? ` (${printable(note)})` : ""}`), }); console.log(`\n${renderLeaderboardMarkdown(result)}`); console.log(` ${path.join(out, "leaderboard.html")}`); @@ -1191,7 +1193,7 @@ program console.log(renderTerminal(result)); const trials = result.tasks.reduce((n, t) => n + t.trials.length, 0); console.log(`\n regraded ${trials} trial(s) without calling a model: ${flipped.length} verdict(s) changed`); - for (const { before, after } of flipped) console.log(` ${STATUS_MARK[trialStatus(before)]} → ${STATUS_MARK[trialStatus(after)]} ${after.taskId} trial ${after.trial + 1}`); + for (const { before, after } of flipped) console.log(` ${STATUS_MARK[trialStatus(before)]} → ${STATUS_MARK[trialStatus(after)]} ${printable(after.taskId)} trial ${after.trial + 1}`); for (const j of [...new Set(["1", String(result.k)])]) { if (stored.passHat[j] !== result.passHat[j]) console.log(` pass^${j} ${formatPercent(stored.passHat[j] ?? 0)} → ${formatPercent(result.passHat[j] ?? 0)}`); } @@ -1219,12 +1221,12 @@ withProviderFlags(program const blocker = judgeBlocker(domain, stored); if (blocker) throw new Error(`can't judge ${resultsPath}: ${blocker}${stored.domain !== domain.name ? `. Pass the --domain the run used: the one whose sandtable.yaml has \`name: ${stored.domain}\`` : ""}`); if (!statesCriteria(domain, stored)) { - console.log(`${domain.name} states no judged criteria for the tasks of this run, so there is nothing to judge.`); + console.log(`${printable(domain.name)} states no judged criteria for the tasks of this run, so there is nothing to judge.`); return; } const claude = { ...providerOpts(o), model: o.model }; const warning = judgeInputsWarning(domain, stored); - if (warning) console.log(` ! ${warning}`); + if (warning) console.log(` ! ${printableLines(warning)}`); const plan = judgePlan(domain, stored, claude); console.log(judgePlanLine(plan)); if (plan.dropped.length) console.log(` ${droppedLine(plan.dropped)}`); @@ -1248,14 +1250,14 @@ withProviderFlags(program claude, concurrency: o.concurrency, onTrial: ({ trial, result, warnings }) => { - const label = `${trial.taskId} trial ${trial.trial + 1}`; + const label = `${printable(trial.taskId)} trial ${trial.trial + 1}`; const tally = (verdict: string, word: string) => { const n = trial.judged!.verdicts.filter((v) => v.verdict === verdict).length; return n ? [`${n} ${word}`] : []; }; const verdicts = [...tally("pass", "pass"), ...tally("fail", "fail"), ...tally("not_applicable", "not applicable")].join(", "); - console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${trial.judged!.error}` : ` ${label}: ${verdicts || "no verdict"}`); - for (const w of warnings) console.log(` ! ${w}`); + console.log(trial.judged!.error !== undefined ? ` ! ${label}: the judge call failed: ${printable(trial.judged!.error)}` : ` ${label}: ${verdicts || "no verdict"}`); + for (const w of warnings) console.log(` ! ${printable(w)}`); writing = writing.then(() => save(result, out, false).then(() => undefined)); return writing; }, @@ -1266,7 +1268,7 @@ withProviderFlags(program throw new Error(`judge stopped: ${outcome.stopped.error}\n${outcome.stopped.notJudged} trial(s) were not judged.${kept}`); } await save(outcome.result, out, o.html); - console.log(`\nsandtable judge · ${outcome.result.domain} · agent ${outcome.result.agent}\n`); + console.log(`\nsandtable judge · ${printable(outcome.result.domain)} · agent ${printable(outcome.result.agent)}\n`); console.log(renderJudged(outcome.result).join("\n")); if (outcome.failed) console.log(`\n ${outcome.failed} of ${outcome.sent} judge call(s) failed, each kept as judged.error on its trial: the judged score leaves them out`); console.log(`\n results: ${results}${o.html ? `\n report: ${path.join(out, "report.html")}` : ""}`); @@ -1282,7 +1284,7 @@ program .action(async (resultsPath: string, o: { labels: string }) => { const results = await readResults(resultsPath, "calibrate"); const calibration = calibrate(results, parseLabels(await readUserText(o.labels, { from: "--labels" }), o.labels)); - console.log(`\nsandtable calibrate · ${results.domain} · agent ${results.agent}\n`); + console.log(`\nsandtable calibrate · ${printable(results.domain)} · agent ${printable(results.agent)}\n`); console.log(renderCalibration(calibration)); }); @@ -1311,6 +1313,8 @@ program }); program.parseAsync().catch((err: unknown) => { - console.error(err instanceof DomainNotFoundError ? err.message : err instanceof DomainError ? `Invalid domain: ${err.message}` : err instanceof Error ? err.message : err); + const message = err instanceof DomainNotFoundError ? err.message : err instanceof DomainError ? `Invalid domain: ${err.message}` : err instanceof Error ? err.message : err; + // An error can quote an agent, a platform, a model's API or a results file: its lines as text, never escape sequences. + console.error(typeof message === "string" ? printableLines(message) : message); process.exit(EXIT_ERROR); }); diff --git a/src/judge/calibrate.ts b/src/judge/calibrate.ts index 0c1a4d9..a1a84cf 100644 --- a/src/judge/calibrate.ts +++ b/src/judge/calibrate.ts @@ -1,5 +1,6 @@ import YAML from "yaml"; import { z } from "zod"; +import { printableCopy } from "../report/printable.ts"; import { formatPercent } from "../report/terminal.ts"; import type { TrialResult } from "../run/episode.ts"; import type { SuiteResult, TaskSummary } from "../run/suite.ts"; @@ -220,8 +221,6 @@ const WHY_NONE: Record = { const rateCell = (r: CalibrationRate) => `${r.rate === null ? "n/a" : formatPercent(r.rate)} (n=${r.n})`; const counted = (n: number, one: string, many: string) => `${n} ${n === 1 ? one : many}`; -/** A quote and a reason are a model's output, and a quote comes from the transcript of the agent under test: one line of text, nothing a terminal would act on. */ -const oneLine = (s: string) => s.replace(/[\s\p{Cc}]+/gu, " ").trim(); function rateTable(m: ModelCalibration): string[] { const outside = (c: CalibrationCounts) => [...(c.notApplicable ? [`${c.notApplicable} not applicable`] : []), ...(c.unlabelled ? [`${c.unlabelled} with no label`] : [])].join(", "); @@ -233,13 +232,16 @@ function rateTable(m: ModelCalibration): string[] { function pairLines(p: LabelledVerdict): string[] { return [ ` ${p.task} trial ${p.trial} · ${p.criterion}: you said ${p.label}, the judge said ${p.verdict}`, - ...(p.quote !== undefined ? [` quote: "${oneLine(p.quote)}"${p.quoteFound === false ? " (not in the transcript)" : ""}`] : []), - ` reason: ${oneLine(p.reason)}`, + ...(p.quote !== undefined ? [` quote: "${p.quote}"${p.quoteFound === false ? " (not in the transcript)" : ""}`] : []), + ` reason: ${p.reason}`, ]; } /** The terminal's account of a calibration: the rates with the pairs behind each, what to read, and what stayed outside. */ -export function renderCalibration(c: Calibration): string { +export function renderCalibration(calibration: Calibration): string { + // A quote and a reason are a model's output, a quote comes from the transcript of the agent under test, and the ids and + // models from a results file: each is shown as one line of text, nothing a terminal would act on. + const c = printableCopy(calibration); const lines = [" How often the judge agrees with your labels, over pairs with both a label and a pass or fail verdict.", " TPR: of the pairs you labelled pass, the share the judge gave pass. TNR: the same for fail. No model was called, and this sets no gate."]; if (c.models.length > 1) lines.push("", ` This file's trials hold ${c.models.length} judge models, so each is reported apart: mixing them would measure neither. To compare models, keep one results file per model (judge -m -o ) and calibrate each.`); if (c.models.length === 0) lines.push("", " No trial holds a verdict, so there is nothing to measure yet: run npx sandtable-ai judge again."); diff --git a/src/report/markdown.ts b/src/report/markdown.ts index e7ffa0b..7ebb113 100644 --- a/src/report/markdown.ts +++ b/src/report/markdown.ts @@ -1,6 +1,7 @@ import type { JudgedCounts } from "../judge/summary.ts"; import { comparisonOf, type Comparison, type TaskChange } from "../run/compare.ts"; import { skippedTrials, tagsBelowFloor, type SuiteResult, type TaskSummary } from "../run/suite.ts"; +import { printable } from "./printable.ts"; import { formatPercent } from "./terminal.ts"; /** @@ -34,11 +35,12 @@ const clip = (s: string, width = CELL) => (s.length <= width ? s : `${s.slice(0, /** * Text a run holds, on one line, so that it renders as itself: it can't break a table or open a tag or a link, and in a * comment it can't notify anyone, link an issue or a pull request, or set math. A URL stays plain text. GitHub can still - * link a commit hash or a CVE id. + * link a commit hash or a CVE id. Control characters, a newline and a tab among them, and bidi controls are spaces, so + * nothing in it acts on a terminal that prints it or shows reversed. */ function escapeMarkdown(s: unknown): string { // As entities, not backslashes: a renderer that ignores a backslash there would still see a tag. - const entities = clip(String(s).replace(/\s*[\r\n]+\s*/g, " ")) + const entities = clip(printable(s)) .replace(/&/g, "&") .replace(//g, ">"); diff --git a/src/report/printable.ts b/src/report/printable.ts new file mode 100644 index 0000000..7eb9590 --- /dev/null +++ b/src/report/printable.ts @@ -0,0 +1,33 @@ +/* + * Text that an agent, a simulated caller, a platform's call or a results file holds, made fit to print: a terminal shows + * it as text, so it can't start an escape sequence, ring the bell, move the cursor or set the window's title, and a + * renderer can't show it reversed. Only what came from outside goes through here: sandtable's own colours are added + * after, so they stay. + */ + +/** C0 controls (a newline and a tab among them), DEL, C1 controls, and the bidi embedding, override and isolate controls. */ +const CONTROL = "\\p{Cc}\\u202a-\\u202e\\u2066-\\u2069"; +const ONE_LINE = new RegExp(`[\\s${CONTROL}]+`, "gu"); +const EACH = new RegExp(`[${CONTROL}]`, "gu"); +/** One character a terminal acts on rather than shows. */ +export const CONTROL_CHAR = new RegExp(`[${CONTROL}]`, "u"); +const LINES = new RegExp(`(?:(?![\\n\\t])[${CONTROL}])+`, "gu"); + +/** On one line, for a value inside a line or a Markdown cell: each run of whitespace and control characters is one space, with none at either end. */ +export function printable(value: unknown): string { + return String(value).replace(ONE_LINE, " ").trim(); +} + +/** Over several lines, for a message that breaks its own lines, such as an error: newlines and tabs stay, and each other run of control characters is one space. */ +export function printableLines(value: string): string { + return value.replace(/\r\n/g, "\n").replace(LINES, " "); +} + +/** A copy of JSON data, such as a run, in which every string is `printable` and every key has each control character as a space, so keys that differ stay apart. */ +export function printableCopy(data: T): T { + if (typeof data === "string") return printable(data) as T; + if (Array.isArray(data)) return data.map(printableCopy) as T; + // fromEntries, not assignment, so a key such as __proto__ stays a key. + if (data !== null && typeof data === "object") return Object.fromEntries(Object.entries(data).map(([k, v]) => [k.replace(EACH, " "), printableCopy(v)])) as T; + return data; +} diff --git a/src/report/terminal.ts b/src/report/terminal.ts index 82b4053..41c448c 100644 --- a/src/report/terminal.ts +++ b/src/report/terminal.ts @@ -3,6 +3,7 @@ import { comparisonOf, type Comparison, type TaskChange } from "../run/compare.t import { gradedOutcomes, skippedTrials, tagsBelowFloor, trialStatus, type SoftScore, type SuiteResult, type TaskSummary } from "../run/suite.ts"; import { costOf, totalTokens, type UsageByModel } from "../run/usage.ts"; import type { JudgedCounts } from "../judge/summary.ts"; +import { printableCopy } from "./printable.ts"; const useColor = process.stdout.isTTY && !process.env.NO_COLOR; const paint = (code: number) => (s: string) => (useColor ? `\x1b[${code}m${s}\x1b[0m` : s); @@ -94,8 +95,8 @@ function taskNote(t: TaskSummary): string { * what the judge's calls used and cost. Empty for a run that was never judged. */ export function renderJudged(r: SuiteResult): string[] { - const j = r.judged; - if (!j) return []; + if (!r.judged) return []; + const j = printableCopy(r.judged); const score = (c: JudgedCounts) => (c.score === null ? "n/a" : formatPercent(c.score)); const tally = (c: JudgedCounts) => [`${c.pass} pass`, `${c.fail} fail`, ...(c.notApplicable ? [`${c.notApplicable} not applicable`] : [])].join(", "); const lines = [bold(" Judged score") + dim(" (a model's judgement: it never changes pass/fail, pass^k, a gate or an exit code)")]; @@ -120,7 +121,11 @@ export function renderJudged(r: SuiteResult): string[] { * file it came from (else the block names the baseline's run id). `withoutJudgedChanges` leaves out how the judged * score changed, for a summary printed before the run is judged. */ -export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withoutJudgedChanges?: boolean } = {}): string { +export function renderTerminal(result: SuiteResult, opts: { baseline?: string; withoutJudgedChanges?: boolean } = {}): string { + // An agent, a caller, a platform or a results file wrote what a run holds: the summary shows a copy that is all text, + // each value on one line, before anything is cut to fit. The comparison is worked out on the run as it is. + const r = printableCopy(result); + const comparison = printableCopy(comparisonOf(result)); const lines: string[] = []; lines.push(bold(`sandtable · ${r.domain}`) + dim(` · agent ${r.agent} · user ${r.user} · k=${r.k}`)); if (r.pendingReview?.length) { @@ -270,7 +275,6 @@ export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withou } } - const comparison = comparisonOf(r); if (comparison) lines.push("", ...renderComparison(comparison, opts.baseline)); if (comparison && !opts.withoutJudgedChanges) lines.push(...spaced(renderJudgedChanges(comparison))); return lines.join("\n"); @@ -278,11 +282,13 @@ export function renderTerminal(r: SuiteResult, opts: { baseline?: string; withou /** * What `sandtable compare` prints: the newer run on two lines, then the block a `run --baseline` summary ends with. - * `r` carries the baseline as `baseline`; `files` names the two results.json files as the command was given them. + * `result` carries the baseline as `baseline`; `files` names the two results.json files as the command was given them. */ -export function renderCompare(r: SuiteResult, files: { baseline: string; results: string }): string { - const comparison = comparisonOf(r); - if (!comparison) throw new Error("renderCompare needs a run with a baseline"); +export function renderCompare(result: SuiteResult, files: { baseline: string; results: string }): string { + const found = comparisonOf(result); + if (!found) throw new Error("renderCompare needs a run with a baseline"); + // As in renderTerminal: a copy that is all text, and the comparison worked out on the run as it is. + const [r, comparison] = [printableCopy(result), printableCopy(found)]; return [ bold(`sandtable compare · ${r.domain} · ${files.results}`), dim(` run ${r.runId} · started ${r.startedAt} · agent ${r.agent} · user ${r.user} · k=${r.k}`), @@ -301,8 +307,8 @@ const spaced = (block: string[]) => (block.length ? ["", ...block] : []); * judged. */ export function renderJudgedChanges(c: Comparison): string[] { - const j = c.judged; - if (!j) return []; + if (!c.judged) return []; + const j = printableCopy(c.judged); const score = (x?: JudgedCounts) => (x === undefined ? "–" : x.score === null ? "n/a" : formatPercent(x.score)); const moved = (b?: JudgedCounts, a?: JudgedCounts) => { if (b === undefined || a === undefined || b.score === null || a.score === null) return ""; diff --git a/src/run/gates.ts b/src/run/gates.ts index 7232607..d302095 100644 --- a/src/run/gates.ts +++ b/src/run/gates.ts @@ -1,3 +1,4 @@ +import { printable } from "../report/printable.ts"; import { formatPercent } from "../report/terminal.ts"; import { criticalFailures, skippedTrials, tagsBelowFloor, type SuiteResult } from "./suite.ts"; @@ -23,7 +24,7 @@ export function runGates(result: SuiteResult, opts: { failUnder?: number } = {}) const trials = result.tasks.reduce((n, t) => n + t.trials.length, 0); failed.push({ status: 2, - message: `${skipped.length} of ${trials} trial(s) ended in an error before the agent broke any rule, so they were skipped and this run is incomplete: rerun it. The first error:\n ${skipped[0]!.error ?? "unknown error"}`, + message: `${skipped.length} of ${trials} trial(s) ended in an error before the agent broke any rule, so they were skipped and this run is incomplete: rerun it. The first error:\n ${printable(skipped[0]!.error ?? "unknown error")}`, }); } const blockers = criticalFailures(result); diff --git a/src/run/leaderboard.ts b/src/run/leaderboard.ts index 98aa546..5f39685 100644 --- a/src/run/leaderboard.ts +++ b/src/run/leaderboard.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { z } from "zod"; import { BRAND_CSS, MARK_SVG } from "../report/brand.ts"; import { renderHtml } from "../report/html.ts"; +import { printableCopy } from "../report/printable.ts"; import type { Domain } from "../spec/schema.ts"; import { noopAgent } from "../sim/noop.ts"; import { scriptedUser } from "../sim/scripted.ts"; @@ -678,7 +679,10 @@ export function softCell(r: LeaderboardRow): string { .join("; "); } -export function renderLeaderboardMarkdown(l: LeaderboardResult): string { +export function renderLeaderboardMarkdown(board: LeaderboardResult): string { + // Labels come from a models file, errors from an agent or its provider, and the rest from the runs: each one line of + // text, with no control character a terminal printing the board would act on. + const l = printableCopy(board); const soft = hasSoft(l); const critical = hasCritical(l); const heads = ["#", "Model", ...passHeads(l.k), ...(critical ? ["Critical"] : []), "Rule violations", "Top failure", ...(soft ? ["Soft rules"] : []), "Cost / trial", "p95 turn"]; diff --git a/src/shell.ts b/src/shell.ts index ae9f86c..ee5dee3 100644 --- a/src/shell.ts +++ b/src/shell.ts @@ -1,8 +1,21 @@ +import { CONTROL_CHAR } from "./report/printable.ts"; + /** * A path as one shell word, so a printed command runs as it is when the path has spaces, quotes or `$`, or starts * with `-`, which the CLI would read as an option. */ export function shellWord(s: string): string { const p = s.startsWith("-") ? `./${s}` : s; + if (CONTROL_CHAR.test(p)) return ansiQuoted(p); return /^[\w./@:+=,-]+$/.test(p) ? p : `'${p.replaceAll("'", `'\\''`)}'`; } + +/** + * A word with a control character in it, such as a file name from someone else's archive, as `$'…'`: bash and zsh read + * it back as the same text, and the terminal that shows the command sees each control character as an escape, not as one. + */ +function ansiQuoted(s: string): string { + // Each UTF-8 byte as \xHH, which bash 3.2 (macOS's) reads too, where \u needs bash 4.2. + const escape = (c: string) => [...Buffer.from(c, "utf8")].map((b) => `\\x${b.toString(16).padStart(2, "0")}`).join(""); + return `$'${[...s].map((c) => (c === "\\" || c === "'" ? `\\${c}` : CONTROL_CHAR.test(c) ? escape(c) : c)).join("")}'`; +} diff --git a/test/control-characters.test.ts b/test/control-characters.test.ts new file mode 100644 index 0000000..0ab98e9 --- /dev/null +++ b/test/control-characters.test.ts @@ -0,0 +1,354 @@ +import { execFile } from "node:child_process"; +import { cp, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { freightScriptedAgent } from "../src/demo/freight-agents.ts"; +import { calibrate, renderCalibration } from "../src/judge/calibrate.ts"; +import type { JudgedSummary } from "../src/judge/summary.ts"; +import { renderMarkdown } from "../src/report/markdown.ts"; +import { renderCompare, renderJudged, renderJudgedChanges, renderTerminal } from "../src/report/terminal.ts"; +import { baselineSnapshot, comparisonOf } from "../src/run/compare.ts"; +import { renderLeaderboardMarkdown, type LeaderboardRow } from "../src/run/leaderboard.ts"; +import { runSuite, type SuiteResult } from "../src/run/suite.ts"; +import { shellWord } from "../src/shell.ts"; +import { httpAgent } from "../src/sim/http.ts"; +import { openaiCompatAgent } from "../src/sim/openai-compat.ts"; +import { scriptedUser } from "../src/sim/scripted.ts"; +import type { AgentAdapter } from "../src/sim/types.ts"; +import { loadDomain } from "../src/spec/load.ts"; + +// What an agent, a simulated caller, a platform or a results file holds reaches the terminal and Markdown as text: +// no escape sequence, bell or C1 control character in it gets through, and sandtable's own colours still do. + +const example = path.resolve("examples/freight"); +const cli = path.resolve("src/cli.ts"); + +/** + * An OSC that sets the window title and ends with a bell, a CSI that clears the screen, a C1 CSI, a colour, a full reset, + * a cursor save, a switch to line drawing, a C1 OSC ended by a C1 ST, a backspace, a carriage return, DEL and NUL, then + * a right-to-left override and an isolate around text. + */ +const CONTROLS = "\x1b]0;a title\x07\x1b[2J\x9b31m\x1b[5m\x1bc\x1b7\x1b(0\x9d0;t\x9c\x08\r\x7f\x00\u202eevil\u2066x\u2069"; +/** CONTROLS without NUL, which no file name can hold, and without the carriage return, backspace and DEL, so a test can name a file with it. */ +const FILE_NAME_CONTROLS = "\x1b]0;a title\x07\x1b[2J\x9b31m\u202eevil\u2066x\u2069"; +/** CONTROLS as a one-line value shows it, after a space. */ +const SHOWN = "]0;a title [2J 31m [5m c 7 (0 0;t evil x"; +/** Every control character but a newline, and the bidi embedding, override and isolate controls. */ +const STRAY = /[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/g; +/** Each control character left in `s` but a newline, with the text around it. */ +const strays = (s: string) => [...s.matchAll(STRAY)].map((m) => JSON.stringify(s.slice(Math.max(0, m.index - 30), m.index + 10))); + +afterEach(() => vi.unstubAllGlobals()); + +const domain = () => loadDomain(example); +const careful = async (): Promise => runSuite({ domain: await domain(), agent: freightScriptedAgent("careful"), user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + +/** A judged summary of one criterion, as `judge` writes it. */ +const judgedSummary = (id: string, model: string): JudgedSummary => ({ + pass: 0, + fail: 1, + notApplicable: 0, + score: 0, + models: [model], + notJudged: 0, + criteria: [{ id, pass: 0, fail: 1, notApplicable: 0, score: 0, pending: true }], + usage: { [model]: { calls: 1, input: 900, output: 90, cacheRead: 0, cacheWrite: 0 } }, + costUsd: null, +}); + +describe("terminal output strips control characters from run values", () => { + it("shows a tool argument in a failure example as text, also where the line is cut to fit", async () => { + // An OSC whose end a cut at 80 columns would leave out, so the rest of the terminal's output would be its title. + const mc = `X\x1b]0;${"t".repeat(120)}\x07${CONTROLS}`; + const agent: AgentAdapter = { + name: "books", + createSession: async () => ({ + reply: async (_message, callTool) => { + await callTool("book_load", { load_id: "L-1001", mc_number: mc, rate: 2100 }); + return "Booked."; + }, + }), + }; + const r = await runSuite({ domain: await domain(), agent, user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + const out = renderTerminal(r); + expect(out).toContain("e.g. book_load referenced carriers X ]0;tttt"); + expect(strays(out)).toEqual([]); + }); + + it("shows an HTTP agent's error body in skipped trials as text", async () => { + vi.stubGlobal("fetch", async () => new Response(`boom ${CONTROLS}`, { status: 500 })); + const r = await runSuite({ domain: await domain(), agent: httpAgent("http://agent.local/step"), user: scriptedUser(), taskIds: ["book-at-posted-rate"] }); + const out = renderTerminal(r); + expect(out).toContain("Skipped trials"); + expect(out).toContain("http://agent.local/step returned 500: boom ]0;a title [2J"); + expect(strays(out)).toEqual([]); + }); + + it("shows an OpenAI-compatible endpoint's error body in skipped trials as text", async () => { + vi.stubGlobal("fetch", async () => new Response(`${CONTROLS} bad request`, { status: 400 })); + const agent = openaiCompatAgent({ baseUrl: "http://llm.local/v1", model: "m" }); + const out = renderTerminal(await runSuite({ domain: await domain(), agent, user: scriptedUser(), taskIds: ["book-at-posted-rate"] })); + expect(out).toContain("http://llm.local/v1/chat/completions returned 400: ]0;a title [2J"); + expect(strays(out)).toEqual([]); + }); + + it("shows the simulated user's sentence in caller notes as text", async () => { + const r = await careful(); + r.tasks[0]!.trials[0]!.callerNotes = [{ turn: 1, why: "talks about its instructions", text: `My instructions say ${CONTROLS}book it.` }]; + const out = renderTerminal(r); + expect(out).toContain(`"My instructions say ${SHOWN} book it."`); + expect(strays(out)).toEqual([]); + }); + + it("shows a results file's agent, task ids, judge models and criteria as text in compare and the judged score", async () => { + const now = await careful(); + now.agent = `agent ${CONTROLS}`; + now.runId = `run ${CONTROLS}`; + now.tasks[0]!.id = `book ${CONTROLS}`; + now.tasks[0]!.passes = 0; + now.tasks[0]!.trials[0]!.judged = { model: `judge ${CONTROLS}`, verdicts: [{ id: `tone ${CONTROLS}`, criterion: `Polite ${CONTROLS}`, verdict: "fail", reason: `Curt ${CONTROLS}`, quote: `No ${CONTROLS}` }] }; + now.judged = judgedSummary(`tone ${CONTROLS}`, `judge ${CONTROLS}`); + const before = structuredClone(now); + before.agent = `old ${CONTROLS}`; + before.user = `caller ${CONTROLS}`; + before.startedAt = `then ${CONTROLS}`; + before.tasks[0]!.passes = 1; + before.tasks[0]!.trials[0]!.judged = { model: `other ${CONTROLS}`, verdicts: [{ id: `tone ${CONTROLS}`, criterion: `Kind ${CONTROLS}`, verdict: "fail", reason: "Curt." }] }; + before.judged = judgedSummary(`tone ${CONTROLS}`, `other ${CONTROLS}`); + const compared: SuiteResult = { ...now, baseline: baselineSnapshot(before) }; + + const outputs = { + compare: renderCompare(compared, { baseline: "baseline.json", results: "results.json" }), + terminal: renderTerminal(compared), + judged: renderJudged(now).join("\n"), + judgedChanges: renderJudgedChanges(comparisonOf(compared)!).join("\n"), + }; + expect(outputs.compare).toContain("broken"); + expect(outputs.compare).toContain(`agent old ${SHOWN} → agent ${SHOWN}`); + expect(outputs.judged).toContain(`by judge ${SHOWN}`); + expect(outputs.judgedChanges).toContain("text differs"); + for (const [name, out] of Object.entries(outputs)) expect(strays(out), name).toEqual([]); + }); + + it("shows a results file's task, criterion, model, quote and reason as text in a calibration", () => { + const [task, criterion, model] = [`a ${CONTROLS}`, `tone ${CONTROLS}`, `judge ${CONTROLS}`]; + const run = { + version: 1, + domain: "freight", + agent: "scripted:careful", + passHat: {}, + tasks: [{ id: task, trials: [{ taskId: task, trial: 0, judged: { model, verdicts: [{ id: criterion, criterion: "Polite.", verdict: "fail", quote: `No ${CONTROLS}`, reason: `Curt ${CONTROLS}` }] } }] }], + judged: judgedSummary(criterion, model), + } as unknown as SuiteResult; + const out = renderCalibration(calibrate(run, [{ task, trial: 1, criterion, label: "pass" }])); + expect(out).toContain(`Judge judge ${SHOWN}`); + expect(out).toContain(`quote: "No ${SHOWN}"`); + expect(strays(out)).toEqual([]); + }); + + it("keeps sandtable's own colours on a terminal, and only those", async () => { + const tty = process.stdout.isTTY; + const noColor = process.env.NO_COLOR; + vi.resetModules(); + process.stdout.isTTY = true; + delete process.env.NO_COLOR; + try { + const terminal = await import("../src/report/terminal.ts"); + const r = await careful(); + r.agent = `agent ${CONTROLS}`; + r.tasks[0]!.passes = 0; + r.tasks[0]!.trials[0]!.callerNotes = [{ turn: 1, why: "talks about its instructions", text: `My instructions say ${CONTROLS}` }]; + const out = terminal.renderTerminal(r); + // Red, green, yellow, bold and dim, each reset after. + expect(out).toContain("\x1b[31m0/1\x1b[0m"); + expect(strays(out.replace(/\x1b\[(?:0|1|2|31|32|33)m/g, ""))).toEqual([]); + } finally { + process.stdout.isTTY = tty; + if (noColor !== undefined) process.env.NO_COLOR = noColor; + vi.resetModules(); + } + }); +}); + +describe("Markdown output strips control characters from run values", () => { + it("report --markdown shows a failure example, the agent, ids and a verdict's reason as text", async () => { + const r = await careful(); + r.agent = `agent ${CONTROLS}`; + r.tasks[0]!.passes = 0; + r.tasks[0]!.trials[0]!.judged = { model: "claude-sonnet-5-5", verdicts: [{ id: "tone", criterion: "Polite.", verdict: "fail", reason: `Curt ${CONTROLS}`, quote: `No ${CONTROLS}` }] }; + r.judged = judgedSummary("tone", "claude-sonnet-5-5"); + r.failureModes = [{ kind: "invariant", id: `rule ${CONTROLS}`, count: 1, tasks: [r.tasks[0]!.id], example: `book_load referenced carriers X${CONTROLS}` }]; + const md = renderMarkdown(r); + expect(md).toContain("Top failure modes"); + expect(md).toContain("Curt \\]0;a title \\[2J 31m \\[5m c 7 \\(0 0;t evil x"); + expect(strays(md)).toEqual([]); + }); + + it("LEADERBOARD.md shows the domain, a model's label, its failure, rule and task ids and an error as text", () => { + const row: LeaderboardRow = { + label: `model ${CONTROLS}`, + agent: "scripted:careful", + baseline: false, + passHat1: 0, + passHatK: 0, + tasks: 1, + trials: 1, + violations: { [`rule ${CONTROLS}`]: 1 }, + topFailure: `rule ${CONTROLS} (1×)`, + soft: { [`soft ${CONTROLS}`]: { applied: 1, broken: 1, rate: 1 } }, + critical: { tasks: 1, failed: [`task ${CONTROLS}`] }, + costUsd: null, + costPerTrialUsd: null, + agentCostPerTrialUsd: null, + agentTurnP95Ms: null, + agentTokens: 0, + }; + const md = renderLeaderboardMarkdown({ + version: 1, + domain: `lanes ${CONTROLS}`, + user: `caller ${CONTROLS}`, + k: 1, + generatedAt: `now ${CONTROLS}`, + sandtable: { version: `1.5.0 ${CONTROLS}` }, + rows: [row, { ...row, label: "crashed", error: `boom ${CONTROLS}` }], + }); + expect(md).toContain(`# Leaderboard: lanes ${SHOWN}`); + expect(md).toContain(`boom ${SHOWN}`); + expect(strays(md)).toEqual([]); + }); +}); + +describe.skipIf(!process.features.typescript)("CLI lines strip control characters from run values", () => { + // No key or AWS setting: a model call, if one slipped through, fails instead of costing money. + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("AWS_") && !["ANTHROPIC_API_KEY", "OPENAI_API_KEY"].includes(k))); + const sandtable = (cwd: string, ...args: string[]) => + promisify(execFile)(process.execPath, [cli, ...args], { env, cwd }).then( + (r) => ({ code: 0, ...r }), + (e: { code: number; stdout: string; stderr: string }) => e, + ); + + it("prints an agent's error as text: every trial ended in one, a skipped trial's gate, and a leaderboard model that failed", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const error = JSON.stringify(`no credentials ${CONTROLS}`); + await writeFile(path.join(dir, "crash.mjs"), `export default { name: "crash", createSession: async () => ({ reply: async () => { throw new Error(${error}); } }) };\n`); + await writeFile( + path.join(dir, "some.mjs"), + `export default { name: "some", createSession: async (ctx) => ({ reply: async () => { if (ctx.sessionId.includes("walk-away")) throw new Error(${error}); return "Sorry, I can't help with that."; } }) };\n`, + ); + await writeFile(path.join(dir, "models.yaml"), "models:\n - { label: crash, agent: ./crash.mjs }\n"); + const run = (agent: string, tasks: string) => sandtable(dir, "run", example, "--agent", agent, "--user", "scripted", "-t", tasks, "--no-html", "-o", path.join(dir, `out-${path.basename(agent, ".mjs")}`)); + const all = await run("./crash.mjs", "book-at-posted-rate"); + expect(all.stderr).toContain("Every trial ended in an error"); + expect(all.stderr).toContain(`no credentials ${SHOWN}`); + const some = await run("./some.mjs", "expired-insurance,walk-away-above-ceiling"); + expect(some.stderr).toContain("ended in an error before the agent broke any rule"); + const board = await sandtable(dir, "leaderboard", example, "-m", "models.yaml", "--user", "scripted", "-k", "1", "-t", "book-at-posted-rate", "-o", path.join(dir, "board")); + expect(board.stdout).toContain("✗ failed"); + expect(board.stdout).toContain(`no credentials ${SHOWN}`); + for (const r of [all, some, board]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("prints a results file's values as text: compare's summary and regression gate, calibrate's header, and errors that name the run's domain", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const now = await careful(); + now.domain = `freight ${CONTROLS}`; + now.agent = `agent ${CONTROLS}`; + now.tasks[0]!.trials[0]!.judged = { model: "claude-sonnet-5-5", verdicts: [{ id: "tone", criterion: "Polite.", verdict: "fail", reason: "Curt." }] }; + now.judged = judgedSummary("tone", "claude-sonnet-5-5"); + const failing = { ...now, tasks: [{ ...now.tasks[0]!, id: `book ${CONTROLS}`, passes: 0 }] }; + const passing = { ...failing, tasks: [{ ...failing.tasks[0]!, passes: 1 }] }; + const other = { ...now, domain: `clinic ${CONTROLS}` }; + for (const [file, r] of Object.entries({ "now.json": now, "failing.json": failing, "passing.json": passing, "other.json": other })) await writeFile(path.join(dir, file), JSON.stringify(r)); + await writeFile(path.join(dir, "labels.json"), JSON.stringify({ labels: [{ task: now.tasks[0]!.id, trial: 1, criterion: "tone", label: "pass" }] })); + const compare = await sandtable(dir, "compare", "passing.json", "failing.json", "--fail-on-regression"); + expect(compare.code).toBe(1); + expect(compare.stdout).toContain(`sandtable compare · freight ${SHOWN}`); + expect(compare.stderr).toContain(`broke since the baseline (--fail-on-regression): book ${SHOWN} 1/1 → 0/1`); + const header = await sandtable(dir, "calibrate", "now.json", "--labels", "labels.json"); + expect(header.stdout).toContain(`sandtable calibrate · freight ${SHOWN} · agent agent ${SHOWN}`); + const errors = [ + await sandtable(dir, "compare", "now.json", "other.json"), + await sandtable(dir, "regrade", "now.json", "-d", example, "-o", path.join(dir, "regraded")), + await sandtable(dir, "judge", "now.json", "-d", example, "--dry-run"), + ]; + expect(errors[0]!.stderr).toContain("now.json and other.json are runs of different domains, freight "); + expect(errors[1]!.stderr).toContain("can't regrade now.json: it is a run of freight "); + expect(errors[2]!.stderr).toContain("can't judge now.json: it is a run of freight "); + for (const r of errors) expect(r.code).toBe(2); + for (const r of [compare, header, ...errors]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("review prints a drafted item's quote as text, keeping its lines", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const copy = path.join(dir, "freight"); + await cp(example, copy, { recursive: true }); + const file = path.join(copy, "sandtable.yaml"); + const yaml = (await readFile(file, "utf8")) + .replace(' source: { file: policy.md, quote: "Ask for', ' review: pending\n source: { file: policy.md, quote: "\\e]0;a title\\a\\e[2J\\x9b31m\\ec\\e7\\e(0\\x9d0;t\\x9c\\b\\r\\x7f\\u202eevil\\u2066\\NAsk for') + .replace(' source: { file: policy.md, quote: "Do **not** book', ' review: pending\n source: { file: policy.md, quote: "Do **not** book'); + await writeFile(file, yaml); + const r = await sandtable(dir, "review", copy); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout).toContain(' " ]0;a title [2J 31m c 7 (0 0;t evil Ask for the carrier'); + expect(r.stdout).toContain('"Do **not** book a carrier if any of these is true:\n - authority type is not `carrier`'); + expect(strays(r.stdout)).toEqual([]); + }); + + it("validate, lint and review print a drafted domain's name and task ids as text", async () => { + // `init` copies a model's ids into tasks.yaml, where YAML keeps an escape as `\e`: it loads back as one. + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const copy = path.join(dir, "freight"); + await cp(example, copy, { recursive: true }); + const spec = path.join(copy, "sandtable.yaml"); + await writeFile(spec, (await readFile(spec, "utf8")).replace("name: freight-carrier-sales", 'name: "freight\\e]0;a title\\a\\x9b2J\\u202eevil"')); + const tasks = path.join(copy, "tasks.yaml"); + await writeFile(tasks, (await readFile(tasks, "utf8")).replace("- id: book-at-posted-rate\n", '- id: "book\\e]0;a title\\a\\x9b2Jat-rate"\n review: pending\n')); + const validate = await sandtable(dir, "validate", copy); + expect(validate.code, validate.stderr).toBe(0); + expect(validate.stdout).toContain("✓ freight ]0;a title 2J evil: "); + const lint = await sandtable(dir, "lint", copy); + expect(lint.stdout).toContain("book ]0;a title 2Jat-rate (write"); + const review = await sandtable(dir, "review", copy); + expect(review.code, review.stderr).toBe(0); + expect(review.stdout).toContain("- task book ]0;a title 2Jat-rate"); + for (const r of [validate, lint, review]) expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("init --transcripts prints a transcript's file name as text", async () => { + // A folder of chats from someone else's archive: its file names are as outside as the chats. + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const chats = path.join(dir, "chats"); + await mkdir(chats); + await cp(example, path.join(dir, "draft"), { recursive: true }); + await writeFile(path.join(chats, `call${FILE_NAME_CONTROLS}.txt`), "Caller: I'd like to book load L-1008 for 2026-03-16.\nAgent: Let me check that load.\n"); + const r = await sandtable(dir, "init", "--transcripts", "chats", "--call-date", "2026-03-14", "-o", "draft", "--dry-run"); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout).toContain(" call ]0;a title [2J 31m evil x .txt: 2026-03-16, stated in the file"); + expect(strays(r.stdout + r.stderr)).toEqual([]); + }); + + it("a printed command quotes a word with control characters so the shell reads it back as it was", async () => { + const word = `chats/call${FILE_NAME_CONTROLS}'s.txt`; + const quoted = shellWord(word); + expect(strays(quoted)).toEqual([]); + const { stdout } = await promisify(execFile)("bash", ["-c", `printf %s ${quoted}`]); + expect(stdout).toBe(word); + }); + + it("init --openapi prints a spec's refs as text in its warnings", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-controls-")); + const schema = { $ref: `./missing${CONTROLS}.yaml` }; + const spec = { + openapi: "3.0.0", + info: { title: "Controls", version: "1" }, + paths: { "/loads": { post: { operationId: "book_load", requestBody: { content: { "application/json": { schema } } }, responses: { "200": { description: "ok" } } } } }, + }; + await writeFile(path.join(dir, "api.json"), JSON.stringify(spec)); + const r = await sandtable(dir, "init", "--openapi", "api.json", "-o", "draft"); + expect(r.code, r.stderr).toBe(0); + expect(r.stdout + r.stderr).toContain("missing ]0;a title [2J 31m"); + expect(strays(r.stdout + r.stderr)).toEqual([]); + }); +}); diff --git a/test/judge.test.ts b/test/judge.test.ts index 0204458..b5b2eed 100644 --- a/test/judge.test.ts +++ b/test/judge.test.ts @@ -549,4 +549,14 @@ describe.skipIf(!process.features.typescript)("the judge command", () => { expect(r.stderr).toContain("2 trial(s) were not judged"); expect(await readFile(f.results, "utf8")).toBe(f.text); }); + + it("prints a failed call's error as text, whether the call failed alone or stopped the judge", { timeout: 30_000 }, async () => { + const f = await fixture(); + const controls = "\x1b]0;a title\x07\x1b[2J\x9b31m\x1bc\x08\r\x7f\u202eevil\u2069"; + const alone = await judgeThrough((await stub(() => ({ status: 400, body: { message: `prompt is too long ${controls}` } }))).base, f.results, "-d", f.domainDir, "-c", "1", "--no-html"); + expect(alone.stdout).toContain("the judge call failed: 400 prompt is too long ]0;a title [2J 31m c evil"); + const stopped = await judgeThrough((await stub(() => ({ status: 401, body: { message: `invalid x-api-key ${controls}` } }))).base, f.results, "-d", f.domainDir, "-c", "1", "--no-html"); + expect(stopped.stderr).toMatch(/judge stopped.*invalid x-api-key/); + for (const r of [alone, stopped]) expect(r.stdout + r.stderr).not.toMatch(/[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/); + }); }); diff --git a/test/serve-driven.test.ts b/test/serve-driven.test.ts index 283382c..127d0e1 100644 --- a/test/serve-driven.test.ts +++ b/test/serve-driven.test.ts @@ -500,6 +500,33 @@ describe("the serve command", () => { expect((await readFile(path.join(dir, "report.html"), "utf8")).length).toBeGreaterThan(1000); }); + it.skipIf(!process.features.typescript)("prints a platform's call id as text, in its live log and its summary", { timeout: 30_000 }, async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "sandtable-serve-cli-")); + const port = await freePort(); + const child = spawn(process.execPath, [cli, "serve", example, "--task", "book-at-posted-rate", "--port", String(port), "-o", dir, "--no-html"], { stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + child.stdout.on("data", (d) => (out += d)); + child.stderr.on("data", (d) => (out += d)); + const url = `http://127.0.0.1:${port}`; + for (let i = 0; i < 100 && !out.includes("Results:"); i++) await new Promise((r) => setTimeout(r, 100)); + // Requests with no call id, and a transcript that lists other calls: the trial's error names the call. + await json(`${url}/carriers/100101`); + await json(`${url}/carriers/100101`); + await post(`${url}/bookings`, BOOK); + const callId = "conv-\x1b]0;a title\x07\x1b[2J\x9b31m\x1bc\x9d0;t\x9c\u202eevil"; + const tools = ["verify_carrier", "book_load"].map((tool_name) => ({ tool_name, params_as_json: "{}", tool_has_been_called: true, type: "webhook" })); + const end = { type: "post_call_transcription", event_timestamp: 1, data: { conversation_id: callId, transcript: [{ role: "agent", message: null, tool_calls: tools }, { role: "agent", message: "L-1001 is booked.", tool_calls: null }] } }; + expect((await post(`${url}/webhook`, end)).body).toMatchObject({ skipped: true }); + const code = await new Promise((resolve) => { + child.on("exit", resolve); + child.kill("SIGINT"); + }); + expect(code).toBe(0); + expect(out).toContain("○ book-at-posted-rate trial 1: runtime (call conv- ]0;a title [2J 31m c 0;t evil's transcript lists its tool calls as verify_carrier, book_load"); + expect(out).toContain("Skipped trials"); + expect(out).not.toMatch(/[\x00-\x09\x0b-\x1f\x7f-\x9f\u202a-\u202e\u2066-\u2069]/); + }); + it.skipIf(!process.features.typescript)("refuses an --idle-timeout that isn't a whole number of minutes", { timeout: 30_000 }, async () => { const results = await Promise.all( ["-1", "abc", "1.5"].map((v) => promisify(execFile)(process.execPath, [cli, "serve", example, "--idle-timeout", v]).catch((err: { code: number; stderr: string }) => err)),