diff --git a/.changeset/clear-beans-read.md b/.changeset/clear-beans-read.md new file mode 100644 index 0000000..b2f7e28 --- /dev/null +++ b/.changeset/clear-beans-read.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add support for defining, running, and reporting multi-turn scenarios. diff --git a/README.md b/README.md index f624b4d..ba832b7 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,21 @@ export default defineScenario({ }) ``` +To test context across a conversation, add follow-up `turns`. Each turn names +the test that must pass before the next prompt is sent: + +```ts +export default defineScenario({ + prompt: 'Change the button color to blue', + turns: [ + { + prompt: 'Actually, make it red instead', + test: 'red.test.ts', + }, + ], +}) +``` + ## Authoring experiments Experiments live in [`./experiments`](./experiments/). Each experiment is a diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 3556a3f..e6cbecd 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -147,6 +147,27 @@ Scenario descriptions and tags are optional. Use `description` to explain what the scenario tests. Pass `tags` to `listScenarios` to return only scenarios that include every requested tag. +### Multi-turn scenarios + +Add `turns` to continue the initial conversation after `scenario.test.ts` +passes. Each follow-up turn provides a prompt and the test file that verifies +the agent's changes. An optional browser test can also run for that turn: + +```ts +export default defineScenario({ + prompt: 'Change the button color to blue', + turns: [ + { + prompt: 'Actually, make it red instead', + test: 'red.test.ts', + browserTest: 'red.browser.test.ts', + }, + ], +}) +``` + +The evaluator stops after a failed turn instead of sending later prompts. + ## Experiment config authoring Use `defineConfig` from `@primer/agent-eval/experiment` to keep local experiment diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts index 6ccbea1..2bcc890 100644 --- a/packages/agent-eval/src/experiment-config.ts +++ b/packages/agent-eval/src/experiment-config.ts @@ -5,6 +5,13 @@ type ScenarioConfig = { description?: string prompt: string tags?: Array + turns?: Array +} + +type ScenarioTurnConfig = { + prompt: string + test: string + browserTest?: string } type InlineScenarioConfig = { @@ -35,4 +42,11 @@ const ControlTreatment: TreatmentConfig = { } export {ControlTreatment} -export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig} +export type { + ExperimentConfig, + ExperimentScenarioConfig, + InlineScenarioConfig, + ScenarioConfig, + ScenarioTurnConfig, + TreatmentConfig, +} diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts index f321971..5f8a540 100644 --- a/packages/agent-eval/src/output.ts +++ b/packages/agent-eval/src/output.ts @@ -107,9 +107,27 @@ const ResolvedScenarioSchema = z.object({ config: z.object({ description: z.optional(z.string()), prompt: z.string(), + turns: z.optional( + z.array( + z.object({ + prompt: z.string(), + test: z.string(), + browserTest: z.optional(z.string()), + }), + ), + ), }), testPath: z.string(), browserTestPath: z.optional(z.string()), + turns: z.optional( + z.array( + z.object({ + prompt: z.string(), + testPath: z.string(), + browserTestPath: z.optional(z.string()), + }), + ), + ), }) const AgentEvalOutputResultSchema = z.object({ diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts index fb9abf6..369f03a 100644 --- a/packages/agent-eval/src/run.test.ts +++ b/packages/agent-eval/src/run.test.ts @@ -32,6 +32,28 @@ describe('getCopilotArgs', () => { 'json', ]) }) + + test('resumes the previous session for a follow-up turn', () => { + expect( + getCopilotArgs({ + prompt: 'Actually, make it red', + model: 'gpt-5.5', + sessionId: 'session-id', + }), + ).toEqual([ + '-p', + 'Actually, make it red', + '--model', + 'gpt-5.5', + '--allow-all', + '--resume', + 'session-id', + '--mode', + 'autopilot', + '--output-format', + 'json', + ]) + }) }) describe('getVitestConfig', () => { diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts index 384a73d..5ef0070 100644 --- a/packages/agent-eval/src/run.ts +++ b/packages/agent-eval/src/run.ts @@ -126,10 +126,12 @@ function getCopilotArgs({ prompt, model, reasoningEffort, + sessionId, }: { prompt: string model: Model reasoningEffort?: ReasoningEffort + sessionId?: string }): Array { const args = ['-p', prompt, '--model', model, '--allow-all'] @@ -137,6 +139,10 @@ function getCopilotArgs({ args.push('--reasoning-effort', reasoningEffort) } + if (sessionId) { + args.push('--resume', sessionId) + } + return [...args, '--mode', 'autopilot', '--output-format', 'json'] } @@ -147,9 +153,20 @@ async function runTreatment( console.log('Running treatment: %s (%s)', treatment.config.name, treatment.id) await using sandbox = await Sandbox.create({dockerImage}) + const scenarioTestPaths = treatment.scenario.turns?.flatMap(turn => [ + path.relative(treatment.scenario.directory, turn.testPath), + ...(turn.browserTestPath ? [path.relative(treatment.scenario.directory, turn.browserTestPath)] : []), + ]) console.log('Copying files from: %s...', treatment.scenario.directory) await sandbox.copy(treatment.scenario.directory, CONTAINER_WORKDIR, { - exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'], + exclude: [ + 'scenario.config.ts', + 'scenario.test.ts', + 'scenario.browser.test.ts', + ...(scenarioTestPaths ?? []), + 'node_modules', + '.next', + ], }) await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], { user: 'root', @@ -189,7 +206,7 @@ async function runTreatment( user: NODE_USER, }) - if (treatment.scenario.browserTestPath) { + if (treatment.scenario.browserTestPath || treatment.scenario.turns?.some(turn => turn.browserTestPath)) { console.log('Installing browser test dependencies...') await sandbox.runCommand( 'npm', @@ -207,50 +224,17 @@ async function runTreatment( }) } - console.log('Running copilot...') - const {prompt} = treatment.scenario.config - const args = getCopilotArgs({ - prompt, - model: treatment.model, - reasoningEffort: treatment.reasoningEffort, - }) - const copilotOutput = await sandbox.runCommand('copilot', args, { - user: NODE_USER, - env: { - COPILOT_GITHUB_TOKEN: copilotToken, - }, - }) - const messages: Array = copilotOutput.stdout.split('\n').flatMap(line => { - const trimmed = line.trim() - if (trimmed.length === 0) { - return [] - } - return parseMessage(JSON.parse(trimmed)) - }) - - const TEST_PATH = 'scenario.test.ts' - const BROWSER_TEST_PATH = 'scenario.browser.test.ts' - const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts' - const TEST_RESULTS_PATH = 'test-results.json' - const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json' - const scenarioTests = [ + const scenarioTurns = [ { - sourcePath: treatment.scenario.testPath, - testPath: TEST_PATH, - resultsPath: TEST_RESULTS_PATH, - browser: false, + prompt: treatment.scenario.config.prompt, + testPath: treatment.scenario.testPath, + ...(treatment.scenario.browserTestPath ? {browserTestPath: treatment.scenario.browserTestPath} : {}), }, + ...(treatment.scenario.turns ?? []), ] - - if (treatment.scenario.browserTestPath) { - scenarioTests.push({ - sourcePath: treatment.scenario.browserTestPath, - testPath: BROWSER_TEST_PATH, - resultsPath: BROWSER_TEST_RESULTS_PATH, - browser: true, - }) - } - + const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts' + const TEST_RESULTS_PATH = 'test-results.json' + const messages: Array = [] let numFailedTests = 0 let numPassedTests = 0 let numPendingTests = 0 @@ -259,53 +243,110 @@ async function runTreatment( let testRunSuccess = true const tests: TreatmentResult['testResults']['tests'] = [] const rawTestResults: Array & {testResults: Array}> = [] + let sessionId: string | undefined + + for (const [turnIndex, turn] of scenarioTurns.entries()) { + console.log('Running copilot turn %d of %d...', turnIndex + 1, scenarioTurns.length) + const args = getCopilotArgs({ + prompt: turn.prompt, + model: treatment.model, + reasoningEffort: treatment.reasoningEffort, + sessionId, + }) + const copilotOutput = await sandbox.runCommand('copilot', args, { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + }) + const turnMessages: Array = copilotOutput.stdout.split('\n').flatMap(line => { + const trimmed = line.trim() + if (trimmed.length === 0) { + return [] + } + return parseMessage(JSON.parse(trimmed)) + }) + messages.push(...turnMessages) - for (const scenarioTest of scenarioTests) { - await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath) - await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser)) - // Always pass vitest calls even if test suite fails - await sandbox.runCommand( - 'sh', - ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath], + const result = turnMessages.find(message => isMessageType(message, 'result')) + if (!result) { + throw new Error(`No result message found in copilot output for turn ${turnIndex + 1}`) + } + sessionId = result.sessionId + + const scenarioTests = [ { - user: NODE_USER, - env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined, + sourcePath: turn.testPath, + testPath: `scenario.turn-${turnIndex + 1}.test.ts`, + resultsPath: `test-results.turn-${turnIndex + 1}.json`, + browser: false, }, - ) + ...(turn.browserTestPath + ? [ + { + sourcePath: turn.browserTestPath, + testPath: `scenario.turn-${turnIndex + 1}.browser.test.ts`, + resultsPath: `browser-test-results.turn-${turnIndex + 1}.json`, + browser: true, + }, + ] + : []), + ] + + for (const scenarioTest of scenarioTests) { + await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath) + await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser)) + // Always pass vitest calls even if test suite fails + await sandbox.runCommand( + 'sh', + ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath], + { + user: NODE_USER, + env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined, + }, + ) + + const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath) + const rawTestResult: unknown = JSON.parse(testResultsContent) + const testResults = parseTestResults(rawTestResult) + if (!testResults.success) { + throw new Error(`Failed to parse test results: ${testResults.error}`) + } - const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath) - const rawTestResult: unknown = JSON.parse(testResultsContent) - const testResults = parseTestResults(rawTestResult) - if (!testResults.success) { - throw new Error(`Failed to parse test results: ${testResults.error}`) + const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8') + numFailedTests += testResults.data.numFailedTests + numPassedTests += testResults.data.numPassedTests + numPendingTests += testResults.data.numPendingTests + numTodoTests += testResults.data.numTodoTests + numTotalTests += testResults.data.numTotalTests + testRunSuccess &&= testResults.data.success + tests.push(...getTestMetadata(testResults.data, testSource)) + rawTestResults.push(rawTestResult as Record & {testResults: Array}) + + await sandbox.runCommand('rm', ['-f', scenarioTest.testPath, scenarioTest.resultsPath], { + user: NODE_USER, + }) } - const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8') - numFailedTests += testResults.data.numFailedTests - numPassedTests += testResults.data.numPassedTests - numPendingTests += testResults.data.numPendingTests - numTodoTests += testResults.data.numTodoTests - numTotalTests += testResults.data.numTotalTests - testRunSuccess &&= testResults.data.success - tests.push(...getTestMetadata(testResults.data, testSource)) - rawTestResults.push(rawTestResult as Record & {testResults: Array}) + await sandbox.runCommand('rm', ['-f', VITEST_CONFIG_PATH], {user: NODE_USER}) + if (!testRunSuccess) { + break + } } - if (rawTestResults.length > 1) { - await sandbox.writeFile( - TEST_RESULTS_PATH, - JSON.stringify({ - ...rawTestResults[0], - numFailedTests, - numPassedTests, - numPendingTests, - numTodoTests, - numTotalTests, - success: testRunSuccess, - testResults: rawTestResults.flatMap(testResult => testResult.testResults), - }), - ) - } + await sandbox.writeFile( + TEST_RESULTS_PATH, + JSON.stringify({ + ...rawTestResults[0], + numFailedTests, + numPassedTests, + numPendingTests, + numTodoTests, + numTotalTests, + success: testRunSuccess, + testResults: rawTestResults.flatMap(testResult => testResult.testResults), + }), + ) // Turns const assistantTurns = new Set() @@ -328,10 +369,7 @@ async function runTreatment( } } - const result = messages.find(message => isMessageType(message, 'result')) - if (!result) { - throw new Error('No result message found in copilot output') - } + const results = messages.filter(message => isMessageType(message, 'result')) const artifactDirectory = path.join(artifactsDirectory, treatment.id) const workspacePath = path.join(artifactDirectory, 'workspace') @@ -367,10 +405,10 @@ async function runTreatment( logs: messages, turns: assistantTurns.size, outputTokens, - premiumRequests: result.usage.premiumRequests, + premiumRequests: results.reduce((total, result) => total + result.usage.premiumRequests, 0), // Time to complete (latency) - totalApiDurationMs: result.usage.totalApiDurationMs, - sessionDurationMs: result.usage.sessionDurationMs, + totalApiDurationMs: results.reduce((total, result) => total + result.usage.totalApiDurationMs, 0), + sessionDurationMs: results.reduce((total, result) => total + result.usage.sessionDurationMs, 0), tools: Object.fromEntries(toolCalls), }, testResults: { diff --git a/packages/agent-eval/src/scenario-config.ts b/packages/agent-eval/src/scenario-config.ts index 0a05641..734eb6d 100644 --- a/packages/agent-eval/src/scenario-config.ts +++ b/packages/agent-eval/src/scenario-config.ts @@ -1,8 +1,8 @@ -import type {ScenarioConfig} from './experiment-config' +import type {ScenarioConfig, ScenarioTurnConfig} from './experiment-config' function defineScenario(config: ScenarioConfig) { return config } export {defineScenario} -export type {ScenarioConfig} +export type {ScenarioConfig, ScenarioTurnConfig} diff --git a/packages/agent-eval/src/scenarios.test.ts b/packages/agent-eval/src/scenarios.test.ts index 0864c0c..b366e82 100644 --- a/packages/agent-eval/src/scenarios.test.ts +++ b/packages/agent-eval/src/scenarios.test.ts @@ -71,7 +71,85 @@ describe('scenario loading', () => { ) await expect(findScenario('example', {directory: scenariosDirectory})).rejects.toThrow( - 'Scenario "example" config must export a default config with a string prompt, optional string description, and optional string[] tags', + 'Scenario "example" config must export a default config with a string prompt, optional string description, optional string[] tags, and optional turns', + ) + }) + + test('loads follow-up turns and their tests', async () => { + const scenariosDirectory = await createScenariosDirectory() + const directory = await createScenario(scenariosDirectory, 'example', 'Make the button blue') + await fs.writeFile( + path.join(directory, 'scenario.config.ts'), + `export default { + prompt: 'Make the button blue', + turns: [{ + prompt: 'Actually, make it red', + test: 'red.test.ts', + browserTest: 'red.browser.test.ts' + }] + }`, + ) + await fs.writeFile(path.join(directory, 'red.test.ts'), '') + await fs.writeFile(path.join(directory, 'red.browser.test.ts'), '') + + await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toMatchObject({ + turns: [ + { + prompt: 'Actually, make it red', + testPath: path.join(directory, 'red.test.ts'), + browserTestPath: path.join(directory, 'red.browser.test.ts'), + }, + ], + }) + }) + + test('allows empty follow-up turns', async () => { + const scenariosDirectory = await createScenariosDirectory() + const directory = await createScenario(scenariosDirectory, 'example', 'Make the button blue') + await fs.writeFile( + path.join(directory, 'scenario.config.ts'), + `export default {prompt: 'Make the button blue', turns: []}`, + ) + + await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({ + id: 'example', + directory, + config: {prompt: 'Make the button blue', turns: []}, + testPath: path.join(directory, 'scenario.test.ts'), + }) + }) + + test('rejects turns without a prompt and test', async () => { + const scenariosDirectory = await createScenariosDirectory() + const directory = await createScenario(scenariosDirectory, 'example', 'Make the button blue') + await fs.writeFile( + path.join(directory, 'scenario.config.ts'), + `export default {prompt: 'Make the button blue', turns: [{prompt: 'Make it red'}]}`, + ) + + await expect(findScenario('example', {directory: scenariosDirectory})).rejects.toThrow( + 'Scenario "example" config must export a default config with a string prompt, optional string description, optional string[] tags, and optional turns', + ) + }) + + test('rejects turn files outside the scenario directory', async () => { + const scenariosDirectory = await createScenariosDirectory() + const directory = await createScenario(scenariosDirectory, 'example', 'Make the button blue') + await fs.writeFile( + path.join(directory, 'scenario.config.ts'), + `export default { + prompt: 'Make the button blue', + turns: [{ + prompt: 'Actually, make it red', + test: 'red.test.ts', + browserTest: '../red.browser.test.ts' + }] + }`, + ) + await fs.writeFile(path.join(directory, 'red.test.ts'), '') + + await expect(findScenario('example', {directory: scenariosDirectory})).rejects.toThrow( + 'Scenario "example" turn file must be a file in the scenario directory: ../red.browser.test.ts', ) }) diff --git a/packages/agent-eval/src/scenarios.ts b/packages/agent-eval/src/scenarios.ts index 18a5686..5657e56 100644 --- a/packages/agent-eval/src/scenarios.ts +++ b/packages/agent-eval/src/scenarios.ts @@ -9,6 +9,13 @@ type ResolvedScenario = { readonly config: ScenarioConfig readonly testPath: string readonly browserTestPath?: string + readonly turns?: ReadonlyArray +} + +type ResolvedScenarioTurn = { + readonly prompt: string + readonly testPath: string + readonly browserTestPath?: string } type ScenarioSourceOptions = { @@ -54,7 +61,20 @@ function isScenarioConfig(value: unknown): value is ScenarioConfig { typeof config.prompt === 'string' && (config.description === undefined || typeof config.description === 'string') && (config.tags === undefined || - (Array.isArray(config.tags) && config.tags.every((tag: unknown) => typeof tag === 'string'))) + (Array.isArray(config.tags) && config.tags.every((tag: unknown) => typeof tag === 'string'))) && + (config.turns === undefined || + (Array.isArray(config.turns) && + config.turns.every((turn: unknown) => { + if (turn === null || typeof turn !== 'object') { + return false + } + const turnConfig = turn as Record + return ( + typeof turnConfig.prompt === 'string' && + typeof turnConfig.test === 'string' && + (turnConfig.browserTest === undefined || typeof turnConfig.browserTest === 'string') + ) + }))) ) } @@ -62,12 +82,22 @@ async function loadScenarioConfig(configPath: string, name: string): Promise { await assertScenarioDirectory(directory, name) @@ -78,12 +108,30 @@ async function loadScenarioDirectory(directory: string, name = path.basename(dir await assertScenarioFile(testPath, name, 'test') const browserTestStats = await fs.stat(browserTestPath).catch(() => undefined) + const config = await loadScenarioConfig(configPath, name) + const turns = await Promise.all( + config.turns?.map(async turn => { + const turnTestPath = resolveScenarioFile(directory, turn.test, name) + await assertScenarioFile(turnTestPath, name, 'test') + const turnBrowserTestPath = turn.browserTest ? resolveScenarioFile(directory, turn.browserTest, name) : undefined + if (turnBrowserTestPath) { + await assertScenarioFile(turnBrowserTestPath, name, 'test') + } + return { + prompt: turn.prompt, + testPath: turnTestPath, + ...(turnBrowserTestPath ? {browserTestPath: turnBrowserTestPath} : {}), + } + }) ?? [], + ) + return { id: name, directory, - config: await loadScenarioConfig(configPath, name), + config, testPath, ...(browserTestStats?.isFile() ? {browserTestPath} : {}), + ...(turns.length > 0 ? {turns} : {}), } } @@ -125,4 +173,4 @@ async function findScenario(id: string, options: ScenarioSourceOptions = {}): Pr } export {findScenario, listScenarios, loadScenarioDirectory} -export type {ResolvedScenario, ScenarioSourceOptions} +export type {ResolvedScenario, ResolvedScenarioTurn, ScenarioSourceOptions} diff --git a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx b/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx index 577a45f..f4b69cd 100644 --- a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx +++ b/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx @@ -31,6 +31,10 @@ type RunResult = { description?: string }> transcript: Array + conversationTurns?: Array<{ + prompt: string + transcript: Array + }> } type RunDetails = { @@ -132,10 +136,37 @@ export function Page({experiment, run}: Props) { ))} -
-

Agent transcript

- -
+ {result.conversationTurns ? ( +
+

Conversation turns

+
    + {result.conversationTurns.map((turn, index) => ( +
  1. +
    + Turn {index + 1} +
    +
    +
    +

    Prompt

    + 
    +                                {turn.prompt}
+
    +
    +
    +

    Agent transcript

    + +
    +
    +
    2. + ))} +
+
+ ) : ( +
+

Agent transcript

+ +
+ )} ))} diff --git a/website/src/app/experiments/[id]/runs/[date]/page.tsx b/website/src/app/experiments/[id]/runs/[date]/page.tsx index 0fb6f40..db86e4a 100644 --- a/website/src/app/experiments/[id]/runs/[date]/page.tsx +++ b/website/src/app/experiments/[id]/runs/[date]/page.tsx @@ -25,7 +25,10 @@ function getString(record: Record | null, key: string): string return typeof value === 'string' ? value : undefined } -function createTranscript(logs: Array): Array { +function createTranscript( + logs: Array, + options: {includeUserMessages?: boolean} = {}, +): Array { const entries: Array = [] const messageEntries = new Map() const reasoningEntries = new Map() @@ -39,6 +42,9 @@ function createTranscript(logs: Array): Array { switch (message.type) { case 'user.message': { + if (options.includeUserMessages === false) { + break + } const content = getString(data, 'content') if (content) { entries.push({id, label: 'User', timestamp, content}) @@ -143,30 +149,62 @@ function createTranscript(logs: Array): Array { return entries.filter(entry => entry.content.length > 0) } +function createConversationTurns(prompts: Array, logs: Array) { + const turnLogs: Array> = [] + let currentTurn: Array = [] + + for (const message of logs) { + currentTurn.push(message) + if (message.type === 'result') { + turnLogs.push(currentTurn) + currentTurn = [] + } + } + + if (currentTurn.length > 0 || turnLogs.length === 0) { + turnLogs.push(currentTurn) + } + + return turnLogs.slice(0, prompts.length).map((messages, index) => ({ + prompt: prompts[index], + transcript: createTranscript(messages, {includeUserMessages: false}), + })) +} + function createRunDetails(date: string, output: AgentEvalOutput): RunDetails { const treatments = new Map(output.treatments.map(treatment => [treatment.id, treatment.config.name])) + const scenarioPrompts = new Map( + output.scenarios.map(scenario => [ + scenario.id, + [scenario.config.prompt, ...(scenario.config.turns?.map(turn => turn.prompt) ?? [])], + ]), + ) return { date, - results: output.results.map(result => ({ - id: result.id, - scenarioId: result.scenarioId, - treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment', - model: result.model, - reasoningEffort: result.reasoningEffort, - testsPassed: result.testResults.numPassedTests, - totalTests: result.testResults.numTotalTests, - turns: result.assistant.turns, - outputTokens: result.assistant.outputTokens, - premiumRequests: result.assistant.premiumRequests, - totalApiDurationMs: result.assistant.totalApiDurationMs, - sessionDurationMs: result.assistant.sessionDurationMs, - tests: result.testResults.tests.map(test => ({ - fullName: test.fullName, - status: test.status, - description: test.description, - })), - transcript: createTranscript(result.assistant.logs), - })), + results: output.results.map(result => { + const prompts = scenarioPrompts.get(result.scenarioId) ?? [] + return { + id: result.id, + scenarioId: result.scenarioId, + treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment', + model: result.model, + reasoningEffort: result.reasoningEffort, + testsPassed: result.testResults.numPassedTests, + totalTests: result.testResults.numTotalTests, + turns: result.assistant.turns, + outputTokens: result.assistant.outputTokens, + premiumRequests: result.assistant.premiumRequests, + totalApiDurationMs: result.assistant.totalApiDurationMs, + sessionDurationMs: result.assistant.sessionDurationMs, + tests: result.testResults.tests.map(test => ({ + fullName: test.fullName, + status: test.status, + description: test.description, + })), + transcript: createTranscript(result.assistant.logs), + conversationTurns: prompts.length > 1 ? createConversationTurns(prompts, result.assistant.logs) : undefined, + } + }), } }