From 54848905a72dabab1eb750cede2026fff68daa7c Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 14 Jul 2026 12:00:21 +0100 Subject: [PATCH] feat(evalboard): show conversation transcript for simulation tasks Simulation runs already write a clean user<->agent transcript to conversation.log (orchestrator._log_conversation), but the evalboard never surfaced it. Add a Conversation section to the task detail page that renders the dialog as chat bubbles, below the Success criteria section. Gated on parsed turns, so single-shot tasks (which never write conversation.log) are unaffected. - lib/runs.ts: readConversationLog (twin of readLogTail) + a pure parseConversation parser returning ConversationTurn[]. - _sections.tsx: ConversationSection renders turns as bubbles (user right/gray, agent left/blue). - page.tsx: fetch + parse, render below Success criteria. Tests mirror repo conventions: unit tests for parseConversation (alongside parseMessages) and a render test for ConversationSection (alongside message-timeline). readConversationLog is left untested, matching its untested twin readLogTail. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[...task]/__tests__/conversation.test.tsx | 40 +++++++++++++ .../app/runs/[id]/[...task]/_sections.tsx | 36 ++++++++++++ evalboard/app/runs/[id]/[...task]/page.tsx | 9 +++ .../lib/__tests__/parseConversation.test.ts | 43 ++++++++++++++ evalboard/lib/runs.ts | 56 +++++++++++++++++++ 5 files changed, 184 insertions(+) create mode 100644 evalboard/app/runs/[id]/[...task]/__tests__/conversation.test.tsx create mode 100644 evalboard/lib/__tests__/parseConversation.test.ts diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/conversation.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/conversation.test.tsx new file mode 100644 index 00000000..53650f93 --- /dev/null +++ b/evalboard/app/runs/[id]/[...task]/__tests__/conversation.test.tsx @@ -0,0 +1,40 @@ +import { describe, expect, test } from "vitest"; +import { render, screen } from "@testing-library/react"; +import type { ConversationTurn } from "@/lib/runs"; +import { ConversationSection } from "../_sections"; + +function makeTurn(overrides: Partial = {}): ConversationTurn { + return { + role: "USER", + turn: 1, + metadata: null, + text: "hello", + ...overrides, + }; +} + +describe("ConversationSection", () => { + test("renders both turns' text and roles", () => { + render( + , + ); + expect(screen.getByText("please write fizzbuzz")).toBeInTheDocument(); + expect(screen.getByText("here is the code")).toBeInTheDocument(); + expect(screen.getByText(/USER · turn 1/)).toBeInTheDocument(); + expect(screen.getByText(/AGENT · turn 1/)).toBeInTheDocument(); + }); + + test("shows metadata in the turn header when present", () => { + render( + , + ); + expect(screen.getByText(/turn 2 · stop_token/)).toBeInTheDocument(); + }); +}); diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index f57ff2e4..6e7f4407 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -3,6 +3,7 @@ import { type ReactNode, useState } from "react"; import type { ArtifactRef, + ConversationTurn, CriterionResult, FlowDebugResult, MessageEvent, @@ -1375,6 +1376,41 @@ export function ArtifactsSection({ ); } +export function ConversationSection({ turns }: { turns: ConversationTurn[] }) { + return ( +
+

Conversation

+
+ {turns.map((t, i) => { + const isUser = t.role === "USER"; + return ( +
+
+
+ {t.role} · turn {t.turn} + {t.metadata ? ` · ${t.metadata}` : ""} +
+
+ {t.text} +
+
+
+ ); + })} +
+
+ ); +} + export function LogTailSection({ log }: { log: string }) { return (
diff --git a/evalboard/app/runs/[id]/[...task]/page.tsx b/evalboard/app/runs/[id]/[...task]/page.tsx index e842da9c..4676d04d 100644 --- a/evalboard/app/runs/[id]/[...task]/page.tsx +++ b/evalboard/app/runs/[id]/[...task]/page.tsx @@ -1,6 +1,8 @@ import Link from "next/link"; import { notFound } from "next/navigation"; import { + parseConversation, + readConversationLog, readLogTail, readTaskDetail, readTaskReplicates, @@ -16,6 +18,7 @@ import { displayedTurns } from "@/lib/turns"; import { ExpectedTurnsStat, TurnsStat } from "./turns-stat"; import { ArtifactsSection, + ConversationSection, CostExplorerSection, CriteriaSection, FlowDebugSection, @@ -59,6 +62,9 @@ export default async function TaskPage({ ); const log = await readLogTail(id, taskId, replicate); + const conversation = parseConversation( + await readConversationLog(id, taskId, replicate), + ); const { flowDebug } = task; return ( @@ -284,6 +290,9 @@ export default async function TaskPage({ {flowDebug && } + {conversation.length > 0 && ( + + )} {task.messages.length > 0 && ( { + test("parses a two-turn dialog", () => { + const raw = [ + "=== USER (turn 1) ===", + "please write fizzbuzz", + "", + "=== AGENT (turn 1) ===", + "here is the code", + "", + ].join("\n"); + + const turns = parseConversation(raw); + + expect(turns).toHaveLength(2); + expect(turns[0]).toEqual({ + role: "USER", + turn: 1, + metadata: null, + text: "please write fizzbuzz", + }); + expect(turns[1].role).toBe("AGENT"); + expect(turns[1].text).toBe("here is the code"); + }); + + test("returns [] for empty input (the non-simulation gate)", () => { + expect(parseConversation("")).toEqual([]); + }); + + test("captures metadata after the em-dash", () => { + const raw = "=== USER (turn 2) — stop_token ===\ndone"; + expect(parseConversation(raw)[0].metadata).toBe("stop_token"); + }); + + test("keeps multi-line bodies intact", () => { + const raw = ["=== AGENT (turn 1) ===", "line one", "line two"].join( + "\n", + ); + expect(parseConversation(raw)[0].text).toBe("line one\nline two"); + }); +}); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 26fa767e..c8538f0d 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -2047,6 +2047,62 @@ export async function readLogTail( return `… (truncated, showing last ${maxBytes} bytes)\n\n${raw.slice(-maxBytes)}`; } +// One user↔agent exchange parsed from conversation.log (simulation runs only). +export interface ConversationTurn { + role: "USER" | "AGENT"; + turn: number; + metadata: string | null; + text: string; +} + +// Reads the clean simulation transcript. Returns "" for non-simulation tasks +// (single-shot runs never write conversation.log) — mirrors readLogTail. +export async function readConversationLog( + runId: string, + taskId: string, + replicate = 0, + maxBytes = 200_000, +): Promise { + await ensureTaskDir(runId, taskId, RUNS_DIR); + const taskDir = taskContentBase(runId, taskId); + const contentDir = await resolveTaskContentDir(taskDir, replicate); + const logPath = path.join(contentDir, "conversation.log"); + const raw = await fs.readFile(logPath, "utf-8").catch(() => ""); + if (raw.length <= maxBytes) return raw; + return `… (truncated, showing last ${maxBytes} bytes)\n\n${raw.slice(-maxBytes)}`; +} + +// Splits conversation.log into ordered turns. The header line is: +// === ROLE (turn N)[ — metadata] === +// Everything up to the next header (or EOF) is that turn's body. +export function parseConversation(raw: string): ConversationTurn[] { + const header = /^=== (USER|AGENT) \(turn (\d+)\)(?: — (.*?))? ===$/; + const turns: ConversationTurn[] = []; + let current: ConversationTurn | null = null; + const body: string[] = []; + const flush = () => { + if (current) current.text = body.join("\n").trim(); + body.length = 0; + }; + for (const line of raw.split("\n")) { + const m = header.exec(line); + if (m) { + flush(); + current = { + role: m[1] as "USER" | "AGENT", + turn: Number(m[2]), + metadata: m[3] ?? null, + text: "", + }; + turns.push(current); + } else if (current) { + body.push(line); + } + } + flush(); + return turns; +} + // Collect every file under a task's folder (`default//`) for the // download-as-zip button on the task page. Reuses walkArtifacts so the same // noise filter (`.venv`, `node_modules`, `*.pyc`, lockfiles, secrets) and