diff --git a/website/README.md b/website/README.md index e8a13ae..6fc98bd 100644 --- a/website/README.md +++ b/website/README.md @@ -2,17 +2,17 @@ ## Routes -| URL | Description | -| :---------------------------- | :---------------------------------------------- | -| `/` | View the latest design system benchmark results | -| `/benchmarks` | List benchmarks | -| `/benchmarks/:id` | View benchmark results and dated runs | -| `/benchmarks/:id/runs/:date` | View benchmark run details and walkthroughs | -| `/experiments` | List experiments | -| `/experiments/:id` | View experiment details | -| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs | -| `/scenarios` | List scenarios | -| `/scenarios/:id` | View scenario details | +| URL | Description | +| :---------------------------- | :--------------------------------------------------- | +| `/` | View the latest benchmark and experiment results | +| `/benchmarks` | List benchmarks | +| `/benchmarks/:id` | View benchmark results and dated runs | +| `/benchmarks/:id/runs/:date` | View benchmark run details and walkthroughs | +| `/experiments` | List experiments | +| `/experiments/:id` | Compare latest treatments, scenarios, and dated runs | +| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs | +| `/scenarios` | List scenarios | +| `/scenarios/:id` | View scenario details | ## Results @@ -31,8 +31,26 @@ results/ Artifact and walkthrough paths are relative to each `output.json`, so result directories should be moved or uploaded as complete bundles. +The overview includes each configured experiment and its latest dated run. +It reads only the newest available result bundle for each experiment; dated +run history is loaded on the experiment page. +Experiment pages compare treatments separately for each model and reasoning +effort, both across the run and within each scenario. Test pass rates use the +sum of passed tests divided by the sum of total tests. Output tokens, premium +requests, session time, and API time are averages per recorded trial. Trial and +scenario counts are shown so differences in coverage are visible; these are +descriptive results, not paired comparisons or significance estimates. + +Select **View output** for a scenario to open its walkthrough, test results, +and transcript. The run viewer supports switching model, treatment, and trial +when multiple trials were recorded. Runs without trials and experiments without +runs show empty states rather than falling back to older results. +Changing the model or treatment selects the first trial for that combination. +Scenario output links also support IDs containing spaces, slashes, and percent +escapes. + Benchmark and experiment run details include a **Judges** tab for the selected -model and treatment. Each judge shows its score, scoring criteria, rationale, +model, treatment, and trial. Each judge shows its score, scoring criteria, rationale, and file-backed findings with code snippets. Scores use the judge's configured scale, not a shared pass/fail threshold. Judge errors and missing results are shown separately from scored results. diff --git a/website/src/app/components/ExperimentResults.tsx b/website/src/app/components/ExperimentResults.tsx new file mode 100644 index 0000000..65c2daf --- /dev/null +++ b/website/src/app/components/ExperimentResults.tsx @@ -0,0 +1,198 @@ +'use client' + +import {Blankslate, DataTable, Table} from '@primer/react/experimental' +import type {Route} from 'next' +import {useId} from 'react' +import type {ExperimentResults, TreatmentResult} from '../../experiment-results' +import {Link} from '../../components/Link' +import {getScenarioAnchor} from '../../scenario-anchor' + +type ExperimentSummary = { + id: string + name: string + description: string + date: string | null + treatments: Array +} + +function formatNumber(value: number): string { + return value.toLocaleString('en-US', {maximumFractionDigits: 1}) +} + +function TreatmentResultsTable({results, label}: {results: Array; label: string}) { + const labelId = useId() + return ( + + + {label} + + { + return row.passRate === null + ? 'N/A (no tests)' + : `${row.passedTests}/${row.totalTests} (${formatNumber(row.passRate * 100)}%)` + }, + }, + { + id: 'tokens', + header: 'Output tokens', + field: 'outputTokens', + align: 'end', + renderCell: row => { + return formatNumber(row.outputTokens) + }, + }, + { + id: 'requests', + header: 'Premium requests', + field: 'premiumRequests', + align: 'end', + renderCell: row => { + return formatNumber(row.premiumRequests) + }, + }, + { + id: 'session', + header: 'Session time', + field: 'sessionDurationMs', + align: 'end', + renderCell: row => { + return `${formatNumber(row.sessionDurationMs / 1000)} s` + }, + }, + { + id: 'api', + header: 'API time', + field: 'totalApiDurationMs', + align: 'end', + renderCell: row => { + return `${formatNumber(row.totalApiDurationMs / 1000)} s` + }, + }, + ]} + data={results} + /> + + ) +} + +function MetricsDescription() { + return ( +

+ Tests passed is the sum of passed tests divided by total tests across recorded trials. Resource usage is the + average per trial. Treatments are grouped by model and reasoning effort; compare scenario and trial counts before + comparing performance. +

+ ) +} + +export function ExperimentsOverview({experiments}: {experiments: Array}) { + return ( +
+
+

+ Experiments +

+ View all experiments +
+ {experiments.length === 0 ?

No experiments have been configured yet.

: } + {experiments.map(experiment => { + return ( +
+

+ {experiment.name} +

+

{experiment.description}

+ {experiment.date ? ( +

+ Latest run:{' '} + + + + . View scenario results and run history +

+ ) : null} + {experiment.treatments.length > 0 ? ( + + ) : ( +

+ {experiment.date + ? 'No trial results were recorded in the latest run.' + : 'No results have been recorded for this experiment yet.'} +

+ )} +
+ ) + })} +
+ ) +} + +export function LatestExperimentResults({id, results}: {id: string; results: ExperimentResults | null}) { + if (!results) { + return null + } + + return ( +
+

+ Latest results +

+

+ Run:{' '} + + + +

+ {results.treatments.length > 0 ? ( + <> + + + + ) : ( + + No trial results + No trial results were recorded in the latest run. + + )} +

Scenario results

+ {results.scenarios.map(scenario => { + return ( +
+

{scenario.id}

+ {scenario.treatments.length > 0 ? ( + <> + +

+ + View output for {scenario.id} + +

+ + ) : ( +

No trial results were recorded for this scenario.

+ )} +
+ ) + })} +
+ ) +} diff --git a/website/src/app/components/RunDetailsPage.tsx b/website/src/app/components/RunDetailsPage.tsx index 4f8c469..f8d0199 100644 --- a/website/src/app/components/RunDetailsPage.tsx +++ b/website/src/app/components/RunDetailsPage.tsx @@ -7,6 +7,7 @@ import type {Route} from 'next' import Link from 'next/link' import Image from 'next/image' import {useState} from 'react' +import {getScenarioAnchor} from '../../scenario-anchor' import {JudgeResults} from './JudgeResults' type RunResult = RunDetails['results'][number] @@ -263,6 +264,7 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num const [selectedModel, setSelectedModel] = useState(getModelValue(group.results[0])) const [selectedTreatment, setSelectedTreatment] = useState(group.results[0].treatment) + const [selectedTrial, setSelectedTrial] = useState(group.results[0].id) const resultsForSelectedModel = group.results.filter(result => { return getModelValue(result) === selectedModel }) @@ -271,18 +273,26 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num return firstTreatment.localeCompare(secondTreatment) }, ) + const activeTreatment = treatmentOptions.includes(selectedTreatment) ? selectedTreatment : treatmentOptions[0] + const trials = resultsForSelectedModel.filter(result => { + return result.treatment === activeTreatment + }) const selectedResult = - resultsForSelectedModel.find(result => { - return result.treatment === selectedTreatment + trials.find(result => { + return result.id === selectedTrial }) ?? - resultsForSelectedModel[0] ?? + trials[0] ?? group.results[0] const resultHeadingId = `result-${index}-heading` const summaryHeadingId = `result-${index}-summary-heading` return ( -
+

{group.scenarioId} @@ -302,9 +312,14 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num }) ? selectedTreatment : (resultsForNextModel[0] ?? group.results[0]).treatment + const nextTrial = + resultsForNextModel.find(result => { + return result.treatment === nextTreatment + }) ?? group.results[0] setSelectedModel(nextModel) setSelectedTreatment(nextTreatment) + setSelectedTrial(nextTrial.id) }} > {sortedModelOptions.map(([value, label]) => { @@ -319,9 +334,16 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num Treatment + {trials.length > 1 ? ( + + Trial + + + ) : null}

diff --git a/website/src/app/components/ScenarioAnchors.test.tsx b/website/src/app/components/ScenarioAnchors.test.tsx new file mode 100644 index 0000000..dcc1685 --- /dev/null +++ b/website/src/app/components/ScenarioAnchors.test.tsx @@ -0,0 +1,59 @@ +import {renderToStaticMarkup} from 'react-dom/server' +import type {Route} from 'next' +import {expect, test} from 'vitest' +import {getExperimentResults} from '../../experiment-results' +import type {RunDetails} from '../../run-details' +import {createResult, createRun} from '../../test/experiment' +import {LatestExperimentResults} from './ExperimentResults' +import {RunDetailsPage} from './RunDetailsPage' + +test.each(['001-button', 'space / literal%20 # caf\u00e9'])( + 'links to the rendered scenario target for %s', + scenarioId => { + const result = createResult({scenarioId}) + const run: RunDetails = { + date: '2026-09-10', + results: [ + { + id: result.id, + scenarioId, + treatment: 'Control', + model: result.model, + reasoningEffort: result.reasoningEffort, + testsPassed: 0, + totalTests: 0, + turns: 0, + outputTokens: 0, + premiumRequests: 0, + totalApiDurationMs: 0, + sessionDurationMs: 0, + tests: [], + transcript: [], + walkthrough: {type: 'Unavailable'}, + judges: [], + }, + ], + } + const overview = renderToStaticMarkup( + , + ) + const details = renderToStaticMarkup( + , + ) + const href = /href="([^"]+#scenario-[^"]+)"/.exec(overview)?.[1] + const target = /]+id="([^"]+)"/.exec(details)?.[1] + expect(href).toBeDefined() + expect(target).toBeDefined() + expect(decodeURIComponent(new URL(href!, 'https://example.test').hash.slice(1))).toBe(target) + expect(target).not.toMatch(/\s/) + }, +) diff --git a/website/src/app/experiments/[id]/components/Page.tsx b/website/src/app/experiments/[id]/components/Page.tsx index 8d544d8..001f9e5 100644 --- a/website/src/app/experiments/[id]/components/Page.tsx +++ b/website/src/app/experiments/[id]/components/Page.tsx @@ -6,6 +6,8 @@ import type {Experiment} from '../../../../experiments' import {Link} from '../../../../components/Link' import type {Route} from 'next' import NextLink from 'next/link' +import type {ExperimentResults} from '../../../../experiment-results' +import {LatestExperimentResults} from '../../../components/ExperimentResults' type ExperimentRun = { id: string @@ -18,9 +20,10 @@ type ExperimentRun = { type Props = { experiment: Experiment runs: Array + results: ExperimentResults | null } -export function Page({experiment, runs}: Props) { +export function Page({experiment, runs, results}: Props) { return ( @@ -33,6 +36,7 @@ export function Page({experiment, runs}: Props) {

{experiment.name}

{experiment.description}

+

Runs

{runs.length > 0 ? ( diff --git a/website/src/app/experiments/[id]/page.tsx b/website/src/app/experiments/[id]/page.tsx index 62f2446..a1c39bd 100644 --- a/website/src/app/experiments/[id]/page.tsx +++ b/website/src/app/experiments/[id]/page.tsx @@ -1,5 +1,5 @@ -import {get, list} from '../../../experiments' -import {listForExperiment} from '../../../runs' +import {list} from '../../../experiments' +import {getExperimentPageData} from '../../../experiment-page-data' import {Page} from './components/Page' type ExperimentPageProps = { @@ -11,19 +11,8 @@ type ExperimentPageProps = { export default async function ExperimentPage(props: ExperimentPageProps) { const params = await props.params const id = params.id - const [experiment, runs] = await Promise.all([get(id), listForExperiment(id)]) - return ( - ({ - id: run.id, - name: run.name, - resultCount: run.output.results.length, - passedTests: run.output.results.reduce((total, result) => total + result.testResults.numPassedTests, 0), - totalTests: run.output.results.reduce((total, result) => total + result.testResults.numTotalTests, 0), - }))} - /> - ) + const data = await getExperimentPageData(id) + return } export async function generateStaticParams() { diff --git a/website/src/app/layout.test.tsx b/website/src/app/layout.test.tsx new file mode 100644 index 0000000..c587b0a --- /dev/null +++ b/website/src/app/layout.test.tsx @@ -0,0 +1,29 @@ +import {renderToStaticMarkup} from 'react-dom/server' +import {expect, test, vi} from 'vitest' +import Layout from './layout' + +vi.mock('next/navigation', () => { + return { + usePathname: () => { + return '/' + }, + } +}) + +test('renders Primer focus-visible markers on the server without suppressing hydration warnings', () => { + const layout = ( + + + + ) + const html = renderToStaticMarkup(layout) + const openingTag = /]*>/.exec(html)?.[0] + + expect(openingTag).toContain('class="js-focus-visible"') + expect(openingTag).toContain('data-js-focus-visible=""') + expect(openingTag).toContain('data-color-mode="auto"') + expect(openingTag).toContain('data-light-theme="light"') + expect(openingTag).toContain('data-dark-theme="dark"') + expect(Layout({children: null}).props.suppressHydrationWarning).not.toBe(true) + expect(html).toContain('Focusable content') +}) diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx index 8f449a1..381e619 100644 --- a/website/src/app/layout.tsx +++ b/website/src/app/layout.tsx @@ -11,8 +11,16 @@ export const metadata = { } export default function Layout({children}: {children: React.ReactNode}) { + // Match the markers Primer's focus-visible polyfill adds before hydration. return ( - + diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx index 2473472..3f190d7 100644 --- a/website/src/app/page.tsx +++ b/website/src/app/page.tsx @@ -1,7 +1,17 @@ import {getBenchmarkPageData} from '../benchmark-page-data' import {BenchmarkOverview} from './components/BenchmarkOverview' +import {getExperimentsOverview} from '../experiment-page-data' +import {ExperimentsOverview} from './components/ExperimentResults' export default async function IndexPage() { - const {benchmark, overview} = await getBenchmarkPageData('design-system') - return + const [{benchmark, overview}, experiments] = await Promise.all([ + getBenchmarkPageData('design-system'), + getExperimentsOverview(), + ]) + return ( + <> + + + + ) } diff --git a/website/src/experiment-page-data.test.ts b/website/src/experiment-page-data.test.ts new file mode 100644 index 0000000..e80d8e1 --- /dev/null +++ b/website/src/experiment-page-data.test.ts @@ -0,0 +1,76 @@ +import {beforeEach, expect, test, vi} from 'vitest' +import {getExperimentPageData, getExperimentsOverview} from './experiment-page-data' +import {get, list} from './experiments' +import {getLatestForExperiment, listForExperiment} from './runs' +import {createResult, createRun} from './test/experiment' + +vi.mock('./experiments', () => { + return {get: vi.fn(), list: vi.fn()} +}) +vi.mock('./runs', () => { + return {getLatestForExperiment: vi.fn(), listForExperiment: vi.fn()} +}) + +const experiment = { + id: 'example', + name: 'Example experiment', + description: 'Compare treatments', + models: [], + scenarios: [], + treatments: [], +} + +beforeEach(() => { + vi.resetAllMocks() + vi.mocked(get).mockResolvedValue(experiment) + vi.mocked(list).mockResolvedValue([experiment]) +}) + +test('uses the newest run from the dated run loader and preserves run history', async () => { + vi.mocked(listForExperiment).mockResolvedValue([ + createRun([createResult()]), + createRun([createResult(), createResult({id: 'second'})], '2026-09-09'), + ]) + const data = await getExperimentPageData('example') + expect(get).toHaveBeenCalledWith('example') + expect(listForExperiment).toHaveBeenCalledWith('example') + expect(data.results).toMatchObject({date: '2026-09-10', treatments: [{trials: 1}]}) + expect(data.runs).toEqual([ + {id: '2026-09-10', name: '2026-09-10', resultCount: 1, passedTests: 3, totalTests: 4}, + {id: '2026-09-09', name: '2026-09-09', resultCount: 2, passedTests: 6, totalTests: 8}, + ]) +}) + +test('overview includes experiments without results and only sends summary data', async () => { + vi.mocked(list).mockResolvedValue([experiment, {...experiment, id: 'empty'}]) + vi.mocked(getLatestForExperiment).mockImplementation(async id => { + return id === 'empty' ? null : createRun() + }) + const overview = await getExperimentsOverview() + expect(getLatestForExperiment).toHaveBeenCalledWith('example') + expect(getLatestForExperiment).toHaveBeenCalledWith('empty') + expect(listForExperiment).not.toHaveBeenCalled() + expect(overview[0]).toMatchObject({id: 'example', date: '2026-09-10', treatments: [{trials: 1}]}) + expect(Object.keys(overview[0])).toEqual(['id', 'name', 'description', 'date', 'treatments']) + expect(overview[1]).toEqual({ + id: 'empty', + name: experiment.name, + description: experiment.description, + date: null, + treatments: [], + }) +}) + +test('does not replace an empty latest run with older results', async () => { + vi.mocked(listForExperiment).mockResolvedValue([createRun([]), createRun(undefined, '2026-09-09')]) + vi.mocked(getLatestForExperiment).mockResolvedValue(createRun([])) + expect((await getExperimentPageData('example')).results).toMatchObject({date: '2026-09-10', treatments: []}) + expect(await getExperimentsOverview()).toMatchObject([{date: '2026-09-10', treatments: []}]) +}) + +test('propagates result loading errors instead of displaying an empty success state', async () => { + vi.mocked(listForExperiment).mockRejectedValue(new Error('Invalid result bundle')) + vi.mocked(getLatestForExperiment).mockRejectedValue(new Error('Invalid result bundle')) + await expect(getExperimentsOverview()).rejects.toThrow('Invalid result bundle') + await expect(getExperimentPageData('example')).rejects.toThrow('Invalid result bundle') +}) diff --git a/website/src/experiment-page-data.ts b/website/src/experiment-page-data.ts new file mode 100644 index 0000000..b1f96f3 --- /dev/null +++ b/website/src/experiment-page-data.ts @@ -0,0 +1,41 @@ +import {get as getExperiment, list as listExperiments} from './experiments' +import {getExperimentResults} from './experiment-results' +import {getLatestForExperiment, listForExperiment} from './runs' + +export async function getExperimentPageData(id: string) { + const [experiment, runs] = await Promise.all([getExperiment(id), listForExperiment(id)]) + return { + experiment, + results: getExperimentResults(runs[0]), + runs: runs.map(run => { + return { + id: run.id, + name: run.name, + resultCount: run.output.results.length, + passedTests: run.output.results.reduce((total, result) => { + return total + result.testResults.numPassedTests + }, 0), + totalTests: run.output.results.reduce((total, result) => { + return total + result.testResults.numTotalTests + }, 0), + } + }), + } +} + +export async function getExperimentsOverview() { + const experiments = await listExperiments() + return Promise.all( + experiments.map(async experiment => { + const run = await getLatestForExperiment(experiment.id) + const results = getExperimentResults(run ?? undefined) + return { + id: experiment.id, + name: experiment.name, + description: experiment.description, + date: results?.date ?? null, + treatments: results?.treatments ?? [], + } + }), + ) +} diff --git a/website/src/experiment-results.test.ts b/website/src/experiment-results.test.ts new file mode 100644 index 0000000..3503fbe --- /dev/null +++ b/website/src/experiment-results.test.ts @@ -0,0 +1,106 @@ +import {expect, test} from 'vitest' +import {getExperimentResults} from './experiment-results' +import {createResult, createRun} from './test/experiment' + +test('distinguishes missing runs from runs with no trials', () => { + expect(getExperimentResults(undefined)).toBeNull() + expect(getExperimentResults(createRun([]))).toEqual({ + date: '2026-09-10', + treatments: [], + scenarios: [ + {id: 'scenario-a', treatments: []}, + {id: 'scenario-b', treatments: []}, + ], + }) +}) + +test('sums test counts and averages resource usage per trial, not per scenario', () => { + const first = createResult() + const second = createResult({ + id: 'trial-2', + assistant: { + ...first.assistant, + outputTokens: 300, + premiumRequests: 3, + sessionDurationMs: 6000, + totalApiDurationMs: 3000, + }, + testResults: {...first.testResults, numPassedTests: 1, numTotalTests: 2}, + }) + const third = createResult({id: 'trial-3', scenarioId: 'scenario-b'}) + const summary = getExperimentResults(createRun([first, second, third])) + + expect(summary?.treatments).toEqual([ + { + id: JSON.stringify(['control', 'gpt-5.6-sol', 'medium']), + treatment: 'Control', + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + trials: 3, + scenarios: 2, + passedTests: 7, + totalTests: 10, + passRate: 0.7, + outputTokens: 500 / 3, + premiumRequests: 5 / 3, + sessionDurationMs: 10000 / 3, + totalApiDurationMs: 5000 / 3, + }, + ]) + expect(summary?.scenarios[0].treatments[0]).toMatchObject({ + trials: 2, + scenarios: 1, + passedTests: 4, + totalTests: 6, + passRate: 4 / 6, + outputTokens: 200, + }) + expect(summary?.scenarios[1].treatments[0]).toMatchObject({trials: 1, passedTests: 3, totalTests: 4}) +}) + +test('keeps treatments, models, and reasoning efforts separate with stable ordering', () => { + const trials = [ + createResult({id: 'skill', treatmentId: 'skill'}), + createResult({id: 'other-model', model: 'gpt-5.6-luna'}), + createResult({id: 'high', reasoningEffort: 'high'}), + createResult({id: 'default', reasoningEffort: undefined}), + createResult(), + ] + const summary = getExperimentResults(createRun(trials)) + expect(summary?.treatments).toHaveLength(5) + expect( + summary?.treatments.every(result => { + return result.trials === 1 + }), + ).toBe(true) + expect( + summary?.treatments.some(result => { + return result.reasoningEffort === 'Default' + }), + ).toBe(true) + expect(getExperimentResults(createRun(trials.toReversed()))).toEqual(summary) +}) + +test('shows no-test results as unavailable rather than zero or perfect performance', () => { + const result = createResult() + result.testResults.numPassedTests = 0 + result.testResults.numTotalTests = 0 + result.assistant.outputTokens = 0 + + expect(getExperimentResults(createRun([result]))?.treatments[0]).toMatchObject({ + trials: 1, + passRate: null, + passedTests: 0, + totalTests: 0, + outputTokens: 0, + }) +}) + +test('preserves recorded scenario and treatment IDs even when metadata is absent', () => { + const result = createResult({treatmentId: 'historical-treatment', scenarioId: 'historical-scenario'}) + const summary = getExperimentResults(createRun([result])) + expect(summary?.scenarios[0]).toMatchObject({ + id: 'historical-scenario', + treatments: [{treatment: 'historical-treatment'}], + }) +}) diff --git a/website/src/experiment-results.ts b/website/src/experiment-results.ts new file mode 100644 index 0000000..b41658c --- /dev/null +++ b/website/src/experiment-results.ts @@ -0,0 +1,117 @@ +import type {Run, RunOutputResult} from './runs' + +export type TreatmentResult = { + id: string + treatment: string + model: string + reasoningEffort: string + trials: number + scenarios: number + passedTests: number + totalTests: number + passRate: number | null + outputTokens: number + premiumRequests: number + sessionDurationMs: number + totalApiDurationMs: number +} + +export type ExperimentResults = { + date: string + treatments: Array + scenarios: Array<{ + id: string + treatments: Array + }> +} + +function summarizeTreatments(run: Run, results: Array): Array { + const groups = new Map>() + const treatments = new Map( + run.output.treatments.map(treatment => { + return [treatment.id, treatment.config.name] + }), + ) + + for (const result of results) { + const key = JSON.stringify([result.treatmentId, result.model, result.reasoningEffort ?? null]) + const group = groups.get(key) + if (group) { + group.push(result) + } else { + groups.set(key, [result]) + } + } + + return Array.from(groups, ([id, trials]) => { + const first = trials[0] + const totals = trials.reduce( + (total, trial) => { + total.passedTests += trial.testResults.numPassedTests + total.totalTests += trial.testResults.numTotalTests + total.outputTokens += trial.assistant.outputTokens + total.premiumRequests += trial.assistant.premiumRequests + total.sessionDurationMs += trial.assistant.sessionDurationMs + total.totalApiDurationMs += trial.assistant.totalApiDurationMs + return total + }, + {passedTests: 0, totalTests: 0, outputTokens: 0, premiumRequests: 0, sessionDurationMs: 0, totalApiDurationMs: 0}, + ) + + return { + id, + treatment: treatments.get(first.treatmentId) ?? first.treatmentId, + model: first.model, + reasoningEffort: first.reasoningEffort ?? 'Default', + trials: trials.length, + scenarios: new Set( + trials.map(trial => { + return trial.scenarioId + }), + ).size, + passedTests: totals.passedTests, + totalTests: totals.totalTests, + passRate: totals.totalTests === 0 ? null : totals.passedTests / totals.totalTests, + outputTokens: totals.outputTokens / trials.length, + premiumRequests: totals.premiumRequests / trials.length, + sessionDurationMs: totals.sessionDurationMs / trials.length, + totalApiDurationMs: totals.totalApiDurationMs / trials.length, + } + }).toSorted((first, second) => { + return ( + first.model.localeCompare(second.model) || + first.reasoningEffort.localeCompare(second.reasoningEffort) || + first.treatment.localeCompare(second.treatment) || + first.id.localeCompare(second.id) + ) + }) +} + +export function getExperimentResults(run: Run | undefined): ExperimentResults | null { + if (!run) { + return null + } + + const scenarios = new Map>() + for (const scenario of run.output.scenarios) { + scenarios.set(scenario.id, []) + } + for (const result of run.output.results) { + const group = scenarios.get(result.scenarioId) + if (group) { + group.push(result) + } else { + scenarios.set(result.scenarioId, [result]) + } + } + + return { + date: run.name, + treatments: summarizeTreatments(run, run.output.results), + scenarios: Array.from(scenarios, ([id, results]) => { + return {id, treatments: summarizeTreatments(run, results)} + }).toSorted((first, second) => { + return first.id.localeCompare(second.id) + }), + } +} diff --git a/website/src/runs.test.ts b/website/src/runs.test.ts new file mode 100644 index 0000000..a8769c7 --- /dev/null +++ b/website/src/runs.test.ts @@ -0,0 +1,104 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import type {ExperimentOutput} from '@primer/agent-eval/experiment' +import {read} from '@primer/agent-eval/experiment' +import {afterEach, beforeEach, expect, test, vi} from 'vitest' + +vi.mock('@primer/agent-eval/experiment', () => { + return {read: vi.fn()} +}) + +let directory: string +let getLatestForExperiment: (typeof import('./runs'))['getLatestForExperiment'] + +beforeEach(async () => { + vi.resetAllMocks() + vi.resetModules() + const temporaryDirectory = path.resolve('.agents/tmp') + await fs.mkdir(temporaryDirectory, {recursive: true}) + directory = await fs.mkdtemp(path.join(temporaryDirectory, 'experiment-runs-')) + const cwd = vi.spyOn(process, 'cwd').mockReturnValue(path.join(directory, 'website')) + try { + getLatestForExperiment = (await import('./runs')).getLatestForExperiment + } finally { + cwd.mockRestore() + } + vi.mocked(read).mockResolvedValue({ + experimentId: 'example', + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + } satisfies ExperimentOutput) +}) + +afterEach(async () => { + await fs.rm(directory, {recursive: true, force: true}) +}) + +async function createRunDirectory(date: string, withOutput = true): Promise { + const runDirectory = path.join(directory, 'results/experiments/example', date) + await fs.mkdir(runDirectory, {recursive: true}) + const outputPath = path.join(runDirectory, 'output.json') + if (withOutput) { + await fs.writeFile(outputPath, '{}') + } + return outputPath +} + +test('reads only the newest available run, including an empty latest run', async () => { + await createRunDirectory('2026-09-08') + const latest = await createRunDirectory('2026-09-10') + await createRunDirectory('2026-09-09') + + expect(await getLatestForExperiment('example')).toMatchObject({ + name: '2026-09-10', + output: {results: []}, + }) + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) +}) + +test('skips invalid dates, files, and directories without a result manifest', async () => { + await createRunDirectory('2026-09-12', false) + await createRunDirectory('2026-13-01') + await createRunDirectory('2026-02-30') + await createRunDirectory('not-a-date') + const latest = await createRunDirectory('2026-09-10') + await fs.writeFile(path.join(directory, 'results/experiments/example/2026-09-11'), '') + + expect(await getLatestForExperiment('example')).toMatchObject({name: '2026-09-10'}) + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) +}) + +test('returns no run when no result bundles exist', async () => { + expect(await getLatestForExperiment('example')).toBeNull() + await createRunDirectory('2026-09-10', false) + expect(await getLatestForExperiment('example')).toBeNull() + expect(read).not.toHaveBeenCalled() +}) + +test('skips a bundle belonging to a different experiment', async () => { + const older = await createRunDirectory('2026-09-09') + const latest = await createRunDirectory('2026-09-10') + vi.mocked(read).mockResolvedValueOnce({ + experimentId: 'other', + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + }) + + expect(await getLatestForExperiment('example')).toMatchObject({name: '2026-09-09'}) + expect(read).toHaveBeenNthCalledWith(1, latest) + expect(read).toHaveBeenNthCalledWith(2, older) +}) + +test('propagates errors in the latest bundle without falling back to older results', async () => { + await createRunDirectory('2026-09-09') + const latest = await createRunDirectory('2026-09-10') + vi.mocked(read).mockRejectedValue(new Error('Invalid result bundle')) + + await expect(getLatestForExperiment('example')).rejects.toThrow('Invalid result bundle') + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) +}) diff --git a/website/src/runs.ts b/website/src/runs.ts index 748b548..25c6c8f 100644 --- a/website/src/runs.ts +++ b/website/src/runs.ts @@ -123,6 +123,21 @@ async function listForExperiment(experimentId: string): Promise> { }) } +async function getLatestForExperiment(experimentId: string): Promise { + const entries = (await listRunDirectories(experimentId)).toSorted((first, second) => { + return second.name.localeCompare(first.name) + }) + + for (const entry of entries) { + const run = await find(experimentId, entry.name) + if (run) { + return run + } + } + + return null +} + async function list(): Promise> { const experiments = await listExperimentDirectories() const runs = await Promise.all( @@ -252,5 +267,5 @@ function normalizeOutput(output: ExperimentOutput): RunOutput { } } -export {list, listForExperiment, get, normalizeOutput} +export {list, listForExperiment, getLatestForExperiment, get, normalizeOutput} export type {Run, RunOutput, RunOutputResult} diff --git a/website/src/scenario-anchor.test.ts b/website/src/scenario-anchor.test.ts new file mode 100644 index 0000000..ab6b4b2 --- /dev/null +++ b/website/src/scenario-anchor.test.ts @@ -0,0 +1,39 @@ +import {expect, test} from 'vitest' +import {getScenarioAnchor} from './scenario-anchor' + +test.each(['001-button', 'with spaces', 'nested/scenario', 'literal%20escape', 'hash#query?', 'caf\u00e9'])( + 'creates a valid target and matching URL fragment for %s', + scenarioId => { + const {id, fragment} = getScenarioAnchor(scenarioId) + expect(id).not.toMatch(/\s/) + expect(id).not.toContain('%') + expect(decodeURIComponent(new URL(fragment, 'https://example.test').hash.slice(1))).toBe(id) + }, +) + +test('keeps existing simple scenario links unchanged', () => { + expect(getScenarioAnchor('001-button')).toEqual({ + id: 'scenario-001-button', + fragment: '#scenario-001-button', + }) +}) + +test('does not confuse percent escapes with the characters they represent', () => { + const anchors = ['a b', 'a%20b', 'a/b', 'a%2Fb', 'a_20b', 'a_2Fb', 'a%2520b'].map(scenarioId => { + return getScenarioAnchor(scenarioId) + }) + expect( + new Set( + anchors.map(anchor => { + return anchor.id + }), + ).size, + ).toBe(anchors.length) + expect( + new Set( + anchors.map(anchor => { + return anchor.fragment + }), + ).size, + ).toBe(anchors.length) +}) diff --git a/website/src/scenario-anchor.ts b/website/src/scenario-anchor.ts new file mode 100644 index 0000000..7748c06 --- /dev/null +++ b/website/src/scenario-anchor.ts @@ -0,0 +1,8 @@ +export function getScenarioAnchor(scenarioId: string): {id: string; fragment: string} { + // Percent-free IDs avoid differences between native and Next.js fragment lookups. + const id = `scenario-${encodeURIComponent(scenarioId).replaceAll('_', '%5F').replaceAll('%', '_')}` + return { + id, + fragment: `#${id}`, + } +} diff --git a/website/src/test/experiment.ts b/website/src/test/experiment.ts new file mode 100644 index 0000000..e8c8181 --- /dev/null +++ b/website/src/test/experiment.ts @@ -0,0 +1,60 @@ +import type {Run, RunOutputResult} from '../runs' + +export function createResult(overrides: Partial = {}): RunOutputResult { + return { + id: 'trial-1', + treatmentId: 'control', + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + scenarioId: 'scenario-a', + assistant: { + logs: [], + turns: 1, + outputTokens: 100, + premiumRequests: 1, + totalApiDurationMs: 1000, + sessionDurationMs: 2000, + tools: {}, + }, + testResults: { + numTotalTests: 4, + numPassedTests: 3, + numFailedTests: 1, + numPendingTests: 0, + numTodoTests: 0, + success: false, + testResults: [], + tests: [], + }, + walkthrough: {type: 'Unavailable'}, + judges: [], + ...overrides, + } +} + +export function createRun(results: Array = [createResult()], date = '2026-09-10'): Run { + return { + id: date, + name: date, + date: new Date(`${date}T00:00:00.000Z`), + directory: `/results/experiments/example/${date}`, + output: { + experiment: {id: 'example', models: []}, + scenarios: ['scenario-b', 'scenario-a'].map(id => { + return { + id, + directory: `/scenarios/${id}`, + prompt: 'Build a page', + tags: [], + judges: [], + testPath: `/scenarios/${id}/scenario.test.ts`, + } + }), + treatments: [ + {id: 'control', config: {name: 'Control'}}, + {id: 'skill', config: {name: 'With skill'}}, + ], + results, + }, + } +} diff --git a/website/vitest.config.ts b/website/vitest.config.ts index d096a6e..60e741b 100644 --- a/website/vitest.config.ts +++ b/website/vitest.config.ts @@ -5,5 +5,10 @@ export default defineConfig({ name: 'website', environment: 'node', include: ['src/**/*.test.{ts,tsx}'], + server: { + deps: { + inline: ['@primer/react'], + }, + }, }, })