From 8656951e3299d2f1ed41a0dd4473a80d69b87aae Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 10 Sep 2026 09:58:03 -0500 Subject: [PATCH 1/3] feat: add experiment results to the overview Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f75a8076-a032-4c34-8135-97855d56fd80 --- vitest.config.ts | 2 +- website/README.md | 35 ++- .../src/app/components/ExperimentResults.tsx | 199 ++++++++++++++++++ website/src/app/components/RunDetailsPage.tsx | 32 ++- .../app/experiments/[id]/components/Page.tsx | 6 +- website/src/app/experiments/[id]/page.tsx | 19 +- website/src/app/page.tsx | 14 +- website/src/experiment-page-data.test.ts | 70 ++++++ website/src/experiment-page-data.ts | 41 ++++ website/src/experiment-results.test.ts | 106 ++++++++++ website/src/experiment-results.ts | 117 ++++++++++ website/src/test/experiment.ts | 58 +++++ website/vitest.config.ts | 9 + 13 files changed, 674 insertions(+), 34 deletions(-) create mode 100644 website/src/app/components/ExperimentResults.tsx create mode 100644 website/src/experiment-page-data.test.ts create mode 100644 website/src/experiment-page-data.ts create mode 100644 website/src/experiment-results.test.ts create mode 100644 website/src/experiment-results.ts create mode 100644 website/src/test/experiment.ts create mode 100644 website/vitest.config.ts diff --git a/vitest.config.ts b/vitest.config.ts index b0cb904..468b5da 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,6 @@ import {defineConfig} from 'vitest/config' export default defineConfig({ test: { passWithNoTests: true, - projects: ['packages/*/vitest.config.ts'], + projects: ['packages/*/vitest.config.ts', 'website/vitest.config.ts'], }, }) diff --git a/website/README.md b/website/README.md index 2ad28f8..11d4aaa 100644 --- a/website/README.md +++ b/website/README.md @@ -2,17 +2,17 @@ ## Routes -| URL | Description | -| :---------------------------- | :---------------------------------------------- | -| `/` | View the latest design system benchmark results | -| `/benchmarks` | List benchmarks | -| `/benchmarks/:id` | View benchmark results and dated runs | -| `/benchmarks/:id/runs/:date` | View benchmark run details and walkthroughs | -| `/experiments` | List experiments | -| `/experiments/:id` | View experiment details | -| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs | -| `/scenarios` | List scenarios | -| `/scenarios/:id` | View scenario details | +| URL | Description | +| :---------------------------- | :--------------------------------------------------- | +| `/` | View the latest benchmark and experiment results | +| `/benchmarks` | List benchmarks | +| `/benchmarks/:id` | View benchmark results and dated runs | +| `/benchmarks/:id/runs/:date` | View benchmark run details and walkthroughs | +| `/experiments` | List experiments | +| `/experiments/:id` | Compare latest treatments, scenarios, and dated runs | +| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs | +| `/scenarios` | List scenarios | +| `/scenarios/:id` | View scenario details | ## Results @@ -30,3 +30,16 @@ results/ Artifact and walkthrough paths are relative to each `output.json`, so result directories should be moved or uploaded as complete bundles. + +The overview includes each configured experiment and its latest dated run. +Experiment pages compare treatments separately for each model and reasoning +effort, both across the run and within each scenario. Test pass rates use the +sum of passed tests divided by the sum of total tests. Output tokens, premium +requests, session time, and API time are averages per recorded trial. Trial and +scenario counts are shown so differences in coverage are visible; these are +descriptive results, not paired comparisons or significance estimates. + +Select **View output** for a scenario to open its walkthrough, test results, +and transcript. The run viewer supports switching model, treatment, and trial +when multiple trials were recorded. Runs without trials and experiments without +runs show empty states rather than falling back to older results. diff --git a/website/src/app/components/ExperimentResults.tsx b/website/src/app/components/ExperimentResults.tsx new file mode 100644 index 0000000..903a42a --- /dev/null +++ b/website/src/app/components/ExperimentResults.tsx @@ -0,0 +1,199 @@ +'use client' + +import {Blankslate, DataTable, Table} from '@primer/react/experimental' +import type {Route} from 'next' +import {useId} from 'react' +import type {ExperimentResults, TreatmentResult} from '../../experiment-results' +import {Link} from '../../components/Link' + +type ExperimentSummary = { + id: string + name: string + description: string + date: string | null + treatments: Array +} + +function formatNumber(value: number): string { + return value.toLocaleString('en-US', {maximumFractionDigits: 1}) +} + +function TreatmentResultsTable({results, label}: {results: Array; label: string}) { + const labelId = useId() + return ( + + + {label} + + { + return row.passRate === null + ? 'N/A (no tests)' + : `${row.passedTests}/${row.totalTests} (${formatNumber(row.passRate * 100)}%)` + }, + }, + { + id: 'tokens', + header: 'Output tokens', + field: 'outputTokens', + align: 'end', + renderCell: row => { + return formatNumber(row.outputTokens) + }, + }, + { + id: 'requests', + header: 'Premium requests', + field: 'premiumRequests', + align: 'end', + renderCell: row => { + return formatNumber(row.premiumRequests) + }, + }, + { + id: 'session', + header: 'Session time', + field: 'sessionDurationMs', + align: 'end', + renderCell: row => { + return `${formatNumber(row.sessionDurationMs / 1000)} s` + }, + }, + { + id: 'api', + header: 'API time', + field: 'totalApiDurationMs', + align: 'end', + renderCell: row => { + return `${formatNumber(row.totalApiDurationMs / 1000)} s` + }, + }, + ]} + data={results} + /> + + ) +} + +function MetricsDescription() { + return ( +

+ Tests passed is the sum of passed tests divided by total tests across recorded trials. Resource usage is the + average per trial. Treatments are grouped by model and reasoning effort; compare scenario and trial counts before + comparing performance. +

+ ) +} + +export function ExperimentsOverview({experiments}: {experiments: Array}) { + return ( +
+
+

+ Experiments +

+ View all experiments +
+ {experiments.length === 0 ?

No experiments have been configured yet.

: } + {experiments.map(experiment => { + return ( +
+

+ {experiment.name} +

+

{experiment.description}

+ {experiment.date ? ( +

+ Latest run:{' '} + + + + . View scenario results and run history +

+ ) : null} + {experiment.treatments.length > 0 ? ( + + ) : ( +

+ {experiment.date + ? 'No trial results were recorded in the latest run.' + : 'No results have been recorded for this experiment yet.'} +

+ )} +
+ ) + })} +
+ ) +} + +export function LatestExperimentResults({id, results}: {id: string; results: ExperimentResults | null}) { + if (!results) { + return null + } + + return ( +
+

+ Latest results +

+

+ Run:{' '} + + + +

+ {results.treatments.length > 0 ? ( + <> + + + + ) : ( + + No trial results + No trial results were recorded in the latest run. + + )} +

Scenario results

+ {results.scenarios.map(scenario => { + return ( +
+

{scenario.id}

+ {scenario.treatments.length > 0 ? ( + <> + +

+ + View output for {scenario.id} + +

+ + ) : ( +

No trial results were recorded for this scenario.

+ )} +
+ ) + })} +
+ ) +} diff --git a/website/src/app/components/RunDetailsPage.tsx b/website/src/app/components/RunDetailsPage.tsx index 28ee1d2..5fee71b 100644 --- a/website/src/app/components/RunDetailsPage.tsx +++ b/website/src/app/components/RunDetailsPage.tsx @@ -248,6 +248,7 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num const [selectedModel, setSelectedModel] = useState(getModelValue(group.results[0])) const [selectedTreatment, setSelectedTreatment] = useState(group.results[0].treatment) + const [selectedTrial, setSelectedTrial] = useState(group.results[0].id) const resultsForSelectedModel = group.results.filter(result => { return getModelValue(result) === selectedModel }) @@ -256,18 +257,22 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num return firstTreatment.localeCompare(secondTreatment) }, ) + const activeTreatment = treatmentOptions.includes(selectedTreatment) ? selectedTreatment : treatmentOptions[0] + const trials = resultsForSelectedModel.filter(result => { + return result.treatment === activeTreatment + }) const selectedResult = - resultsForSelectedModel.find(result => { - return result.treatment === selectedTreatment + trials.find(result => { + return result.id === selectedTrial }) ?? - resultsForSelectedModel[0] ?? + trials[0] ?? group.results[0] const resultHeadingId = `result-${index}-heading` const summaryHeadingId = `result-${index}-summary-heading` return ( -
+

{group.scenarioId} @@ -318,6 +323,25 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num })} + {trials.length > 1 ? ( + + Trial + + + ) : null}

diff --git a/website/src/app/experiments/[id]/components/Page.tsx b/website/src/app/experiments/[id]/components/Page.tsx index 8d544d8..001f9e5 100644 --- a/website/src/app/experiments/[id]/components/Page.tsx +++ b/website/src/app/experiments/[id]/components/Page.tsx @@ -6,6 +6,8 @@ import type {Experiment} from '../../../../experiments' import {Link} from '../../../../components/Link' import type {Route} from 'next' import NextLink from 'next/link' +import type {ExperimentResults} from '../../../../experiment-results' +import {LatestExperimentResults} from '../../../components/ExperimentResults' type ExperimentRun = { id: string @@ -18,9 +20,10 @@ type ExperimentRun = { type Props = { experiment: Experiment runs: Array + results: ExperimentResults | null } -export function Page({experiment, runs}: Props) { +export function Page({experiment, runs, results}: Props) { return ( @@ -33,6 +36,7 @@ export function Page({experiment, runs}: Props) {

{experiment.name}

{experiment.description}

+

Runs

{runs.length > 0 ? ( diff --git a/website/src/app/experiments/[id]/page.tsx b/website/src/app/experiments/[id]/page.tsx index 62f2446..a1c39bd 100644 --- a/website/src/app/experiments/[id]/page.tsx +++ b/website/src/app/experiments/[id]/page.tsx @@ -1,5 +1,5 @@ -import {get, list} from '../../../experiments' -import {listForExperiment} from '../../../runs' +import {list} from '../../../experiments' +import {getExperimentPageData} from '../../../experiment-page-data' import {Page} from './components/Page' type ExperimentPageProps = { @@ -11,19 +11,8 @@ type ExperimentPageProps = { export default async function ExperimentPage(props: ExperimentPageProps) { const params = await props.params const id = params.id - const [experiment, runs] = await Promise.all([get(id), listForExperiment(id)]) - return ( - ({ - id: run.id, - name: run.name, - resultCount: run.output.results.length, - passedTests: run.output.results.reduce((total, result) => total + result.testResults.numPassedTests, 0), - totalTests: run.output.results.reduce((total, result) => total + result.testResults.numTotalTests, 0), - }))} - /> - ) + const data = await getExperimentPageData(id) + return } export async function generateStaticParams() { diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx index 2473472..3f190d7 100644 --- a/website/src/app/page.tsx +++ b/website/src/app/page.tsx @@ -1,7 +1,17 @@ import {getBenchmarkPageData} from '../benchmark-page-data' import {BenchmarkOverview} from './components/BenchmarkOverview' +import {getExperimentsOverview} from '../experiment-page-data' +import {ExperimentsOverview} from './components/ExperimentResults' export default async function IndexPage() { - const {benchmark, overview} = await getBenchmarkPageData('design-system') - return + const [{benchmark, overview}, experiments] = await Promise.all([ + getBenchmarkPageData('design-system'), + getExperimentsOverview(), + ]) + return ( + <> + + + + ) } diff --git a/website/src/experiment-page-data.test.ts b/website/src/experiment-page-data.test.ts new file mode 100644 index 0000000..f077744 --- /dev/null +++ b/website/src/experiment-page-data.test.ts @@ -0,0 +1,70 @@ +import {beforeEach, expect, test, vi} from 'vitest' +import {getExperimentPageData, getExperimentsOverview} from './experiment-page-data' +import {get, list} from './experiments' +import {listForExperiment} from './runs' +import {createResult, createRun} from './test/experiment' + +vi.mock('./experiments', () => { + return {get: vi.fn(), list: vi.fn()} +}) +vi.mock('./runs', () => { + return {listForExperiment: vi.fn()} +}) + +const experiment = { + id: 'example', + name: 'Example experiment', + description: 'Compare treatments', + models: [], + scenarios: [], + treatments: [], +} + +beforeEach(() => { + vi.resetAllMocks() + vi.mocked(get).mockResolvedValue(experiment) + vi.mocked(list).mockResolvedValue([experiment]) +}) + +test('uses the newest run from the dated run loader and preserves run history', async () => { + vi.mocked(listForExperiment).mockResolvedValue([ + createRun([createResult()]), + createRun([createResult(), createResult({id: 'second'})], '2026-09-09'), + ]) + const data = await getExperimentPageData('example') + expect(get).toHaveBeenCalledWith('example') + expect(listForExperiment).toHaveBeenCalledWith('example') + expect(data.results).toMatchObject({date: '2026-09-10', treatments: [{trials: 1}]}) + expect(data.runs).toEqual([ + {id: '2026-09-10', name: '2026-09-10', resultCount: 1, passedTests: 3, totalTests: 4}, + {id: '2026-09-09', name: '2026-09-09', resultCount: 2, passedTests: 6, totalTests: 8}, + ]) +}) + +test('overview includes experiments without results and only sends summary data', async () => { + vi.mocked(list).mockResolvedValue([experiment, {...experiment, id: 'empty'}]) + vi.mocked(listForExperiment).mockImplementation(async id => { + return id === 'empty' ? [] : [createRun()] + }) + const overview = await getExperimentsOverview() + expect(overview[0]).toMatchObject({id: 'example', date: '2026-09-10', treatments: [{trials: 1}]}) + expect(Object.keys(overview[0])).toEqual(['id', 'name', 'description', 'date', 'treatments']) + expect(overview[1]).toEqual({ + id: 'empty', + name: experiment.name, + description: experiment.description, + date: null, + treatments: [], + }) +}) + +test('does not replace an empty latest run with older results', async () => { + vi.mocked(listForExperiment).mockResolvedValue([createRun([]), createRun(undefined, '2026-09-09')]) + expect((await getExperimentPageData('example')).results).toMatchObject({date: '2026-09-10', treatments: []}) +}) + +test('propagates result loading errors instead of displaying an empty success state', async () => { + vi.mocked(listForExperiment).mockRejectedValue(new Error('Invalid result bundle')) + await expect(getExperimentsOverview()).rejects.toThrow('Invalid result bundle') + await expect(getExperimentPageData('example')).rejects.toThrow('Invalid result bundle') +}) diff --git a/website/src/experiment-page-data.ts b/website/src/experiment-page-data.ts new file mode 100644 index 0000000..9942e10 --- /dev/null +++ b/website/src/experiment-page-data.ts @@ -0,0 +1,41 @@ +import {get as getExperiment, list as listExperiments} from './experiments' +import {getExperimentResults} from './experiment-results' +import {listForExperiment} from './runs' + +export async function getExperimentPageData(id: string) { + const [experiment, runs] = await Promise.all([getExperiment(id), listForExperiment(id)]) + return { + experiment, + results: getExperimentResults(runs[0]), + runs: runs.map(run => { + return { + id: run.id, + name: run.name, + resultCount: run.output.results.length, + passedTests: run.output.results.reduce((total, result) => { + return total + result.testResults.numPassedTests + }, 0), + totalTests: run.output.results.reduce((total, result) => { + return total + result.testResults.numTotalTests + }, 0), + } + }), + } +} + +export async function getExperimentsOverview() { + const experiments = await listExperiments() + return Promise.all( + experiments.map(async experiment => { + const runs = await listForExperiment(experiment.id) + const results = getExperimentResults(runs[0]) + return { + id: experiment.id, + name: experiment.name, + description: experiment.description, + date: results?.date ?? null, + treatments: results?.treatments ?? [], + } + }), + ) +} diff --git a/website/src/experiment-results.test.ts b/website/src/experiment-results.test.ts new file mode 100644 index 0000000..3503fbe --- /dev/null +++ b/website/src/experiment-results.test.ts @@ -0,0 +1,106 @@ +import {expect, test} from 'vitest' +import {getExperimentResults} from './experiment-results' +import {createResult, createRun} from './test/experiment' + +test('distinguishes missing runs from runs with no trials', () => { + expect(getExperimentResults(undefined)).toBeNull() + expect(getExperimentResults(createRun([]))).toEqual({ + date: '2026-09-10', + treatments: [], + scenarios: [ + {id: 'scenario-a', treatments: []}, + {id: 'scenario-b', treatments: []}, + ], + }) +}) + +test('sums test counts and averages resource usage per trial, not per scenario', () => { + const first = createResult() + const second = createResult({ + id: 'trial-2', + assistant: { + ...first.assistant, + outputTokens: 300, + premiumRequests: 3, + sessionDurationMs: 6000, + totalApiDurationMs: 3000, + }, + testResults: {...first.testResults, numPassedTests: 1, numTotalTests: 2}, + }) + const third = createResult({id: 'trial-3', scenarioId: 'scenario-b'}) + const summary = getExperimentResults(createRun([first, second, third])) + + expect(summary?.treatments).toEqual([ + { + id: JSON.stringify(['control', 'gpt-5.6-sol', 'medium']), + treatment: 'Control', + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + trials: 3, + scenarios: 2, + passedTests: 7, + totalTests: 10, + passRate: 0.7, + outputTokens: 500 / 3, + premiumRequests: 5 / 3, + sessionDurationMs: 10000 / 3, + totalApiDurationMs: 5000 / 3, + }, + ]) + expect(summary?.scenarios[0].treatments[0]).toMatchObject({ + trials: 2, + scenarios: 1, + passedTests: 4, + totalTests: 6, + passRate: 4 / 6, + outputTokens: 200, + }) + expect(summary?.scenarios[1].treatments[0]).toMatchObject({trials: 1, passedTests: 3, totalTests: 4}) +}) + +test('keeps treatments, models, and reasoning efforts separate with stable ordering', () => { + const trials = [ + createResult({id: 'skill', treatmentId: 'skill'}), + createResult({id: 'other-model', model: 'gpt-5.6-luna'}), + createResult({id: 'high', reasoningEffort: 'high'}), + createResult({id: 'default', reasoningEffort: undefined}), + createResult(), + ] + const summary = getExperimentResults(createRun(trials)) + expect(summary?.treatments).toHaveLength(5) + expect( + summary?.treatments.every(result => { + return result.trials === 1 + }), + ).toBe(true) + expect( + summary?.treatments.some(result => { + return result.reasoningEffort === 'Default' + }), + ).toBe(true) + expect(getExperimentResults(createRun(trials.toReversed()))).toEqual(summary) +}) + +test('shows no-test results as unavailable rather than zero or perfect performance', () => { + const result = createResult() + result.testResults.numPassedTests = 0 + result.testResults.numTotalTests = 0 + result.assistant.outputTokens = 0 + + expect(getExperimentResults(createRun([result]))?.treatments[0]).toMatchObject({ + trials: 1, + passRate: null, + passedTests: 0, + totalTests: 0, + outputTokens: 0, + }) +}) + +test('preserves recorded scenario and treatment IDs even when metadata is absent', () => { + const result = createResult({treatmentId: 'historical-treatment', scenarioId: 'historical-scenario'}) + const summary = getExperimentResults(createRun([result])) + expect(summary?.scenarios[0]).toMatchObject({ + id: 'historical-scenario', + treatments: [{treatment: 'historical-treatment'}], + }) +}) diff --git a/website/src/experiment-results.ts b/website/src/experiment-results.ts new file mode 100644 index 0000000..b41658c --- /dev/null +++ b/website/src/experiment-results.ts @@ -0,0 +1,117 @@ +import type {Run, RunOutputResult} from './runs' + +export type TreatmentResult = { + id: string + treatment: string + model: string + reasoningEffort: string + trials: number + scenarios: number + passedTests: number + totalTests: number + passRate: number | null + outputTokens: number + premiumRequests: number + sessionDurationMs: number + totalApiDurationMs: number +} + +export type ExperimentResults = { + date: string + treatments: Array + scenarios: Array<{ + id: string + treatments: Array + }> +} + +function summarizeTreatments(run: Run, results: Array): Array { + const groups = new Map>() + const treatments = new Map( + run.output.treatments.map(treatment => { + return [treatment.id, treatment.config.name] + }), + ) + + for (const result of results) { + const key = JSON.stringify([result.treatmentId, result.model, result.reasoningEffort ?? null]) + const group = groups.get(key) + if (group) { + group.push(result) + } else { + groups.set(key, [result]) + } + } + + return Array.from(groups, ([id, trials]) => { + const first = trials[0] + const totals = trials.reduce( + (total, trial) => { + total.passedTests += trial.testResults.numPassedTests + total.totalTests += trial.testResults.numTotalTests + total.outputTokens += trial.assistant.outputTokens + total.premiumRequests += trial.assistant.premiumRequests + total.sessionDurationMs += trial.assistant.sessionDurationMs + total.totalApiDurationMs += trial.assistant.totalApiDurationMs + return total + }, + {passedTests: 0, totalTests: 0, outputTokens: 0, premiumRequests: 0, sessionDurationMs: 0, totalApiDurationMs: 0}, + ) + + return { + id, + treatment: treatments.get(first.treatmentId) ?? first.treatmentId, + model: first.model, + reasoningEffort: first.reasoningEffort ?? 'Default', + trials: trials.length, + scenarios: new Set( + trials.map(trial => { + return trial.scenarioId + }), + ).size, + passedTests: totals.passedTests, + totalTests: totals.totalTests, + passRate: totals.totalTests === 0 ? null : totals.passedTests / totals.totalTests, + outputTokens: totals.outputTokens / trials.length, + premiumRequests: totals.premiumRequests / trials.length, + sessionDurationMs: totals.sessionDurationMs / trials.length, + totalApiDurationMs: totals.totalApiDurationMs / trials.length, + } + }).toSorted((first, second) => { + return ( + first.model.localeCompare(second.model) || + first.reasoningEffort.localeCompare(second.reasoningEffort) || + first.treatment.localeCompare(second.treatment) || + first.id.localeCompare(second.id) + ) + }) +} + +export function getExperimentResults(run: Run | undefined): ExperimentResults | null { + if (!run) { + return null + } + + const scenarios = new Map>() + for (const scenario of run.output.scenarios) { + scenarios.set(scenario.id, []) + } + for (const result of run.output.results) { + const group = scenarios.get(result.scenarioId) + if (group) { + group.push(result) + } else { + scenarios.set(result.scenarioId, [result]) + } + } + + return { + date: run.name, + treatments: summarizeTreatments(run, run.output.results), + scenarios: Array.from(scenarios, ([id, results]) => { + return {id, treatments: summarizeTreatments(run, results)} + }).toSorted((first, second) => { + return first.id.localeCompare(second.id) + }), + } +} diff --git a/website/src/test/experiment.ts b/website/src/test/experiment.ts new file mode 100644 index 0000000..591c3a1 --- /dev/null +++ b/website/src/test/experiment.ts @@ -0,0 +1,58 @@ +import type {Run, RunOutputResult} from '../runs' + +export function createResult(overrides: Partial = {}): RunOutputResult { + return { + id: 'trial-1', + treatmentId: 'control', + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + scenarioId: 'scenario-a', + assistant: { + logs: [], + turns: 1, + outputTokens: 100, + premiumRequests: 1, + totalApiDurationMs: 1000, + sessionDurationMs: 2000, + tools: {}, + }, + testResults: { + numTotalTests: 4, + numPassedTests: 3, + numFailedTests: 1, + numPendingTests: 0, + numTodoTests: 0, + success: false, + testResults: [], + tests: [], + }, + walkthrough: {type: 'Unavailable'}, + ...overrides, + } +} + +export function createRun(results: Array = [createResult()], date = '2026-09-10'): Run { + return { + id: date, + name: date, + date: new Date(`${date}T00:00:00.000Z`), + directory: `/results/experiments/example/${date}`, + output: { + experiment: {id: 'example', models: []}, + scenarios: ['scenario-b', 'scenario-a'].map(id => { + return { + id, + directory: `/scenarios/${id}`, + prompt: 'Build a page', + tags: [], + testPath: `/scenarios/${id}/scenario.test.ts`, + } + }), + treatments: [ + {id: 'control', config: {name: 'Control'}}, + {id: 'skill', config: {name: 'With skill'}}, + ], + results, + }, + } +} diff --git a/website/vitest.config.ts b/website/vitest.config.ts new file mode 100644 index 0000000..dde7cc3 --- /dev/null +++ b/website/vitest.config.ts @@ -0,0 +1,9 @@ +import {defineConfig} from 'vitest/config' + +export default defineConfig({ + test: { + name: 'website', + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) From 6259a763c11bf43463ab4d3d2480523d810f9f04 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 10 Sep 2026 11:27:45 -0500 Subject: [PATCH 2/3] fix: make scenario result anchors consistent Share fragment and target generation, cover encoded scenario IDs, and simplify run-loader mock assertions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f75a8076-a032-4c34-8135-97855d56fd80 --- website/README.md | 2 + .../src/app/components/ExperimentResults.tsx | 5 +- website/src/app/components/RunDetailsPage.tsx | 7 ++- .../app/components/ScenarioAnchors.test.tsx | 59 +++++++++++++++++++ website/src/runs.test.ts | 9 ++- website/src/scenario-anchor.test.ts | 39 ++++++++++++ website/src/scenario-anchor.ts | 8 +++ website/vitest.config.ts | 5 ++ 8 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 website/src/app/components/ScenarioAnchors.test.tsx create mode 100644 website/src/scenario-anchor.test.ts create mode 100644 website/src/scenario-anchor.ts diff --git a/website/README.md b/website/README.md index 6c99bf2..6fc98bd 100644 --- a/website/README.md +++ b/website/README.md @@ -46,6 +46,8 @@ and transcript. The run viewer supports switching model, treatment, and trial when multiple trials were recorded. Runs without trials and experiments without runs show empty states rather than falling back to older results. Changing the model or treatment selects the first trial for that combination. +Scenario output links also support IDs containing spaces, slashes, and percent +escapes. Benchmark and experiment run details include a **Judges** tab for the selected model, treatment, and trial. Each judge shows its score, scoring criteria, rationale, diff --git a/website/src/app/components/ExperimentResults.tsx b/website/src/app/components/ExperimentResults.tsx index 903a42a..65c2daf 100644 --- a/website/src/app/components/ExperimentResults.tsx +++ b/website/src/app/components/ExperimentResults.tsx @@ -5,6 +5,7 @@ import type {Route} from 'next' import {useId} from 'react' import type {ExperimentResults, TreatmentResult} from '../../experiment-results' import {Link} from '../../components/Link' +import {getScenarioAnchor} from '../../scenario-anchor' type ExperimentSummary = { id: string @@ -180,9 +181,7 @@ export function LatestExperimentResults({id, results}: {id: string; results: Exp

View output for {scenario.id} diff --git a/website/src/app/components/RunDetailsPage.tsx b/website/src/app/components/RunDetailsPage.tsx index 0a8d107..f8d0199 100644 --- a/website/src/app/components/RunDetailsPage.tsx +++ b/website/src/app/components/RunDetailsPage.tsx @@ -7,6 +7,7 @@ import type {Route} from 'next' import Link from 'next/link' import Image from 'next/image' import {useState} from 'react' +import {getScenarioAnchor} from '../../scenario-anchor' import {JudgeResults} from './JudgeResults' type RunResult = RunDetails['results'][number] @@ -287,7 +288,11 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num const summaryHeadingId = `result-${index}-summary-heading` return ( -

+

{group.scenarioId} diff --git a/website/src/app/components/ScenarioAnchors.test.tsx b/website/src/app/components/ScenarioAnchors.test.tsx new file mode 100644 index 0000000..dcc1685 --- /dev/null +++ b/website/src/app/components/ScenarioAnchors.test.tsx @@ -0,0 +1,59 @@ +import {renderToStaticMarkup} from 'react-dom/server' +import type {Route} from 'next' +import {expect, test} from 'vitest' +import {getExperimentResults} from '../../experiment-results' +import type {RunDetails} from '../../run-details' +import {createResult, createRun} from '../../test/experiment' +import {LatestExperimentResults} from './ExperimentResults' +import {RunDetailsPage} from './RunDetailsPage' + +test.each(['001-button', 'space / literal%20 # caf\u00e9'])( + 'links to the rendered scenario target for %s', + scenarioId => { + const result = createResult({scenarioId}) + const run: RunDetails = { + date: '2026-09-10', + results: [ + { + id: result.id, + scenarioId, + treatment: 'Control', + model: result.model, + reasoningEffort: result.reasoningEffort, + testsPassed: 0, + totalTests: 0, + turns: 0, + outputTokens: 0, + premiumRequests: 0, + totalApiDurationMs: 0, + sessionDurationMs: 0, + tests: [], + transcript: [], + walkthrough: {type: 'Unavailable'}, + judges: [], + }, + ], + } + const overview = renderToStaticMarkup( + , + ) + const details = renderToStaticMarkup( + , + ) + const href = /href="([^"]+#scenario-[^"]+)"/.exec(overview)?.[1] + const target = /]+id="([^"]+)"/.exec(details)?.[1] + expect(href).toBeDefined() + expect(target).toBeDefined() + expect(decodeURIComponent(new URL(href!, 'https://example.test').hash.slice(1))).toBe(target) + expect(target).not.toMatch(/\s/) + }, +) diff --git a/website/src/runs.test.ts b/website/src/runs.test.ts index 183a8e6..a8769c7 100644 --- a/website/src/runs.test.ts +++ b/website/src/runs.test.ts @@ -54,7 +54,8 @@ test('reads only the newest available run, including an empty latest run', async name: '2026-09-10', output: {results: []}, }) - expect(read).toHaveBeenCalledExactlyOnceWith(latest) + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) }) test('skips invalid dates, files, and directories without a result manifest', async () => { @@ -66,7 +67,8 @@ test('skips invalid dates, files, and directories without a result manifest', as await fs.writeFile(path.join(directory, 'results/experiments/example/2026-09-11'), '') expect(await getLatestForExperiment('example')).toMatchObject({name: '2026-09-10'}) - expect(read).toHaveBeenCalledExactlyOnceWith(latest) + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) }) test('returns no run when no result bundles exist', async () => { @@ -97,5 +99,6 @@ test('propagates errors in the latest bundle without falling back to older resul vi.mocked(read).mockRejectedValue(new Error('Invalid result bundle')) await expect(getLatestForExperiment('example')).rejects.toThrow('Invalid result bundle') - expect(read).toHaveBeenCalledExactlyOnceWith(latest) + expect(read).toHaveBeenCalledTimes(1) + expect(read).toHaveBeenCalledWith(latest) }) diff --git a/website/src/scenario-anchor.test.ts b/website/src/scenario-anchor.test.ts new file mode 100644 index 0000000..ab6b4b2 --- /dev/null +++ b/website/src/scenario-anchor.test.ts @@ -0,0 +1,39 @@ +import {expect, test} from 'vitest' +import {getScenarioAnchor} from './scenario-anchor' + +test.each(['001-button', 'with spaces', 'nested/scenario', 'literal%20escape', 'hash#query?', 'caf\u00e9'])( + 'creates a valid target and matching URL fragment for %s', + scenarioId => { + const {id, fragment} = getScenarioAnchor(scenarioId) + expect(id).not.toMatch(/\s/) + expect(id).not.toContain('%') + expect(decodeURIComponent(new URL(fragment, 'https://example.test').hash.slice(1))).toBe(id) + }, +) + +test('keeps existing simple scenario links unchanged', () => { + expect(getScenarioAnchor('001-button')).toEqual({ + id: 'scenario-001-button', + fragment: '#scenario-001-button', + }) +}) + +test('does not confuse percent escapes with the characters they represent', () => { + const anchors = ['a b', 'a%20b', 'a/b', 'a%2Fb', 'a_20b', 'a_2Fb', 'a%2520b'].map(scenarioId => { + return getScenarioAnchor(scenarioId) + }) + expect( + new Set( + anchors.map(anchor => { + return anchor.id + }), + ).size, + ).toBe(anchors.length) + expect( + new Set( + anchors.map(anchor => { + return anchor.fragment + }), + ).size, + ).toBe(anchors.length) +}) diff --git a/website/src/scenario-anchor.ts b/website/src/scenario-anchor.ts new file mode 100644 index 0000000..7748c06 --- /dev/null +++ b/website/src/scenario-anchor.ts @@ -0,0 +1,8 @@ +export function getScenarioAnchor(scenarioId: string): {id: string; fragment: string} { + // Percent-free IDs avoid differences between native and Next.js fragment lookups. + const id = `scenario-${encodeURIComponent(scenarioId).replaceAll('_', '%5F').replaceAll('%', '_')}` + return { + id, + fragment: `#${id}`, + } +} diff --git a/website/vitest.config.ts b/website/vitest.config.ts index d096a6e..60e741b 100644 --- a/website/vitest.config.ts +++ b/website/vitest.config.ts @@ -5,5 +5,10 @@ export default defineConfig({ name: 'website', environment: 'node', include: ['src/**/*.test.{ts,tsx}'], + server: { + deps: { + inline: ['@primer/react'], + }, + }, }, }) From 6cb0076f0b1a94ca50ccfe80c8efbb90763804f3 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 10 Sep 2026 11:35:11 -0500 Subject: [PATCH 3/3] fix: align root markup with focus-visible hydration Render Primer's deterministic focus-visible markers in the root layout so pre-hydration polyfill initialization does not cause an attribute mismatch. Add server-rendering coverage without suppressing hydration warnings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f75a8076-a032-4c34-8135-97855d56fd80 --- website/src/app/layout.test.tsx | 29 +++++++++++++++++++++++++++++ website/src/app/layout.tsx | 10 +++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 website/src/app/layout.test.tsx diff --git a/website/src/app/layout.test.tsx b/website/src/app/layout.test.tsx new file mode 100644 index 0000000..c587b0a --- /dev/null +++ b/website/src/app/layout.test.tsx @@ -0,0 +1,29 @@ +import {renderToStaticMarkup} from 'react-dom/server' +import {expect, test, vi} from 'vitest' +import Layout from './layout' + +vi.mock('next/navigation', () => { + return { + usePathname: () => { + return '/' + }, + } +}) + +test('renders Primer focus-visible markers on the server without suppressing hydration warnings', () => { + const layout = ( + + + + ) + const html = renderToStaticMarkup(layout) + const openingTag = /]*>/.exec(html)?.[0] + + expect(openingTag).toContain('class="js-focus-visible"') + expect(openingTag).toContain('data-js-focus-visible=""') + expect(openingTag).toContain('data-color-mode="auto"') + expect(openingTag).toContain('data-light-theme="light"') + expect(openingTag).toContain('data-dark-theme="dark"') + expect(Layout({children: null}).props.suppressHydrationWarning).not.toBe(true) + expect(html).toContain('Focusable content') +}) diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx index 8f449a1..381e619 100644 --- a/website/src/app/layout.tsx +++ b/website/src/app/layout.tsx @@ -11,8 +11,16 @@ export const metadata = { } export default function Layout({children}: {children: React.ReactNode}) { + // Match the markers Primer's focus-visible polyfill adds before hydration. return ( - +