feat(evalboard): show conversation transcript for simulation tasks

evalboard/app/runs/[id]/[...task]/__tests__/conversation.test.tsx

@@ -0,0 +1,40 @@
+import { describe, expect, test } from "vitest";
+import { render, screen } from "@testing-library/react";
+import type { ConversationTurn } from "@/lib/runs";
+import { ConversationSection } from "../_sections";
+
+function makeTurn(overrides: Partial<ConversationTurn> = {}): ConversationTurn {
+    return {
+        role: "USER",
+        turn: 1,
+        metadata: null,
+        text: "hello",
+        ...overrides,
+    };
+}
+
+describe("ConversationSection", () => {
+    test("renders both turns' text and roles", () => {
+        render(
+            <ConversationSection
+                turns={[
+                    makeTurn({ role: "USER", text: "please write fizzbuzz" }),
+                    makeTurn({ role: "AGENT", text: "here is the code" }),
+                ]}
+            />,
+        );
+        expect(screen.getByText("please write fizzbuzz")).toBeInTheDocument();
+        expect(screen.getByText("here is the code")).toBeInTheDocument();
+        expect(screen.getByText(/USER · turn 1/)).toBeInTheDocument();
+        expect(screen.getByText(/AGENT · turn 1/)).toBeInTheDocument();
+    });
+
+    test("shows metadata in the turn header when present", () => {
+        render(
+            <ConversationSection
+                turns={[makeTurn({ turn: 2, metadata: "stop_token" })]}
+            />,
+        );
+        expect(screen.getByText(/turn 2 · stop_token/)).toBeInTheDocument();
+    });
+});

evalboard/app/runs/[id]/[...task]/_sections.tsx

@@ -3,6 +3,7 @@
 import { type ReactNode, useState } from "react";
 import type {
     ArtifactRef,
+    ConversationTurn,
     CriterionResult,
     FlowDebugResult,
     MessageEvent,
@@ -1375,6 +1376,41 @@ export function ArtifactsSection({
     );
 }
 
+export function ConversationSection({ turns }: { turns: ConversationTurn[] }) {
+    return (
+        <section className="space-y-2">
+            <h2 className="text-sm font-semibold text-gray-900">Conversation</h2>
+            <div className="space-y-3">
+                {turns.map((t, i) => {
+                    const isUser = t.role === "USER";
+                    return (
+                        <div
+                            key={i}
+                            className={`flex ${isUser ? "justify-end" : "justify-start"}`}
+                        >
+                            <div
+                                className={`max-w-[80%] rounded-lg border p-3 ${
+                                    isUser
+                                        ? "bg-gray-50 border-gray-200"
+                                        : "bg-studio-blue/5 border-studio-blue/20"
+                                }`}
+                            >
+                                <div className="text-[10px] uppercase tracking-wide text-gray-500 mb-1">
+                                    {t.role} · turn {t.turn}
+                                    {t.metadata ? ` · ${t.metadata}` : ""}
+                                </div>
+                                <div className="whitespace-pre-wrap text-sm text-gray-800 leading-relaxed">
+                                    {t.text}
+                                </div>
+                            </div>
+                        </div>
+                    );
+                })}
+            </div>
+        </section>
+    );
+}
+
 export function LogTailSection({ log }: { log: string }) {
     return (
         <section className="space-y-2">

evalboard/app/runs/[id]/[...task]/page.tsx

@@ -1,6 +1,8 @@
 import Link from "next/link";
 import { notFound } from "next/navigation";
 import {
+    parseConversation,
+    readConversationLog,
     readLogTail,
     readTaskDetail,
     readTaskReplicates,
@@ -16,6 +18,7 @@ import { displayedTurns } from "@/lib/turns";
 import { ExpectedTurnsStat, TurnsStat } from "./turns-stat";
 import {
     ArtifactsSection,
+    ConversationSection,
     CostExplorerSection,
     CriteriaSection,
     FlowDebugSection,
@@ -59,6 +62,9 @@ export default async function TaskPage({
     );
 
     const log = await readLogTail(id, taskId, replicate);
+    const conversation = parseConversation(
+        await readConversationLog(id, taskId, replicate),
+    );
     const { flowDebug } = task;
 
     return (
@@ -284,6 +290,9 @@ export default async function TaskPage({
 
             {flowDebug && <FlowDebugSection flowDebug={flowDebug} />}
             <CriteriaSection criteria={task.criteria} />
+            {conversation.length > 0 && (
+                <ConversationSection turns={conversation} />
+            )}
             {task.messages.length > 0 && (
                 <CostExplorerSection
                     messages={task.messages}

evalboard/lib/__tests__/parseConversation.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "vitest";
+import { parseConversation } from "../runs";
+
+describe("parseConversation", () => {
+    test("parses a two-turn dialog", () => {
+        const raw = [
+            "=== USER (turn 1) ===",
+            "please write fizzbuzz",
+            "",
+            "=== AGENT (turn 1) ===",
+            "here is the code",
+            "",
+        ].join("\n");
+
+        const turns = parseConversation(raw);
+
+        expect(turns).toHaveLength(2);
+        expect(turns[0]).toEqual({
+            role: "USER",
+            turn: 1,
+            metadata: null,
+            text: "please write fizzbuzz",
+        });
+        expect(turns[1].role).toBe("AGENT");
+        expect(turns[1].text).toBe("here is the code");
+    });
+
+    test("returns [] for empty input (the non-simulation gate)", () => {
+        expect(parseConversation("")).toEqual([]);
+    });
+
+    test("captures metadata after the em-dash", () => {
+        const raw = "=== USER (turn 2) — stop_token ===\ndone";
+        expect(parseConversation(raw)[0].metadata).toBe("stop_token");
+    });
+
+    test("keeps multi-line bodies intact", () => {
+        const raw = ["=== AGENT (turn 1) ===", "line one", "line two"].join(
+            "\n",
+        );
+        expect(parseConversation(raw)[0].text).toBe("line one\nline two");
+    });
+});

evalboard/lib/runs.ts

@@ -2047,6 +2047,62 @@ export async function readLogTail(
     return `… (truncated, showing last ${maxBytes} bytes)\n\n${raw.slice(-maxBytes)}`;
 }
 
+// One user↔agent exchange parsed from conversation.log (simulation runs only).
+export interface ConversationTurn {
+    role: "USER" | "AGENT";
+    turn: number;
+    metadata: string | null;
+    text: string;
+}
+
+// Reads the clean simulation transcript. Returns "" for non-simulation tasks
+// (single-shot runs never write conversation.log) — mirrors readLogTail.
+export async function readConversationLog(
+    runId: string,
+    taskId: string,
+    replicate = 0,
+    maxBytes = 200_000,
+): Promise<string> {
+    await ensureTaskDir(runId, taskId, RUNS_DIR);
+    const taskDir = taskContentBase(runId, taskId);
+    const contentDir = await resolveTaskContentDir(taskDir, replicate);
+    const logPath = path.join(contentDir, "conversation.log");
+    const raw = await fs.readFile(logPath, "utf-8").catch(() => "");
+    if (raw.length <= maxBytes) return raw;
+    return `… (truncated, showing last ${maxBytes} bytes)\n\n${raw.slice(-maxBytes)}`;
+}
+
+// Splits conversation.log into ordered turns. The header line is:
+//   === ROLE (turn N)[ — metadata] ===
+// Everything up to the next header (or EOF) is that turn's body.
+export function parseConversation(raw: string): ConversationTurn[] {
+    const header = /^=== (USER|AGENT) \(turn (\d+)\)(?: — (.*?))? ===$/;
+    const turns: ConversationTurn[] = [];
+    let current: ConversationTurn | null = null;
+    const body: string[] = [];
+    const flush = () => {
+        if (current) current.text = body.join("\n").trim();
+        body.length = 0;
+    };
+    for (const line of raw.split("\n")) {
+        const m = header.exec(line);
+        if (m) {
+            flush();
+            current = {
+                role: m[1] as "USER" | "AGENT",
+                turn: Number(m[2]),
+                metadata: m[3] ?? null,
+                text: "",
+            };
+            turns.push(current);
+        } else if (current) {
+            body.push(line);
+        }
+    }
+    flush();
+    return turns;
+}
+
 // Collect every file under a task's folder (`default/<taskId>/`) for the
 // download-as-zip button on the task page. Reuses walkArtifacts so the same
 // noise filter (`.venv`, `node_modules`, `*.pyc`, lockfiles, secrets) and