From c2185b7d716334971a7eb85285dff58ef8c9598e Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 20:41:21 -0500 Subject: [PATCH 1/7] feat: add durable plan sharding Add reusable plan artifacts, deterministic shard execution, result merging, and four-way benchmark and experiment workflows. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6d12a97a-b8d3-45ad-a09a-3c76d1411705 --- .changeset/bright-plans-shard.md | 5 + .github/workflows/benchmark.yml | 92 +++++++- .github/workflows/experiment.yml | 102 ++++++++- packages/agent-eval/README.md | 36 ++++ packages/agent-eval/package.json | 4 + packages/agent-eval/rolldown.config.ts | 1 + packages/agent-eval/src/benchmark.test.ts | 106 ++++++++++ packages/agent-eval/src/benchmark.ts | 221 +++++++++++++++++--- packages/agent-eval/src/cli-options.test.ts | 77 +++++++ packages/agent-eval/src/cli-options.ts | 149 +++++++++++++ packages/agent-eval/src/cli.test.ts | 21 +- packages/agent-eval/src/cli.ts | 200 ++++++++++++++++-- packages/agent-eval/src/experiment.test.ts | 176 +++++++++++++++- packages/agent-eval/src/experiment.ts | 203 +++++++++++++++--- packages/agent-eval/src/index.test.ts | 6 + packages/agent-eval/src/index.ts | 28 +++ packages/agent-eval/src/output.test.ts | 160 ++++++++++++++ packages/agent-eval/src/output.ts | 174 +++++++++++++++ packages/agent-eval/src/plan.test.ts | 82 +++++++- packages/agent-eval/src/plan.ts | 149 +++++++++++-- script/run-benchmark.sh | 58 ++++- script/run-experiment.sh | 58 ++++- 22 files changed, 1987 insertions(+), 121 deletions(-) create mode 100644 .changeset/bright-plans-shard.md create mode 100644 packages/agent-eval/src/cli-options.test.ts create mode 100644 packages/agent-eval/src/cli-options.ts create mode 100644 packages/agent-eval/src/output.test.ts create mode 100644 packages/agent-eval/src/output.ts diff --git a/.changeset/bright-plans-shard.md b/.changeset/bright-plans-shard.md new file mode 100644 index 00000000..2cd53377 --- /dev/null +++ b/.changeset/bright-plans-shard.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add durable plan creation, replay, deterministic sharding, and shard output merging to the CLI and public package API. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0c721635..8f5ce6bd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,8 +25,47 @@ permissions: contents: read jobs: + plan: + runs-on: ubuntu-latest + outputs: + run-date: ${{ steps.run-date.outputs.value }} + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: set run date + id: run-date + run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT" + - name: create benchmark plan + env: + RUN_DATE: ${{ steps.run-date.outputs.value }} + run: script/run-benchmark.sh design-system plan + - name: upload benchmark plan + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-plan-${{ github.run_id }} + path: results + if-no-files-found: error + retention-days: 1 + compression-level: 9 + run: + needs: plan runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + shard: [1, 2, 3, 4] steps: - name: checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -41,21 +80,66 @@ jobs: run: pnpm install --frozen-lockfile - name: build project run: pnpm run build - - name: run benchmark + - name: download benchmark plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmark-plan-${{ github.run_id }} + - name: run benchmark shard env: CONCURRENCY: ${{ inputs.concurrency }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - run: script/run-benchmark.sh design-system - - name: prepare benchmark artifact + RUN_DATE: ${{ needs.plan.outputs.run-date }} + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + run: script/run-benchmark.sh design-system shard + - name: prepare benchmark shard artifact if: ${{ always() }} run: | mkdir -p workflow-artifact if [[ -d results ]]; then mv results workflow-artifact/results fi - - name: upload benchmark results + - name: upload benchmark shard if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-shard-${{ github.run_id }}-${{ matrix.shard }} + path: workflow-artifact + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + merge: + needs: [plan, run] + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: download benchmark shards + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: benchmark-shard-${{ github.run_id }}-* + merge-multiple: true + - name: merge benchmark shards + env: + RUN_DATE: ${{ needs.plan.outputs.run-date }} + run: script/run-benchmark.sh design-system merge + - name: prepare benchmark artifact + run: | + mkdir -p workflow-artifact + mv results workflow-artifact/results + - name: upload benchmark results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: benchmark-${{ github.run_id }} diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 95437416..b54961f4 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -27,37 +27,121 @@ permissions: contents: read jobs: - run: + plan: runs-on: ubuntu-latest + outputs: + run-date: ${{ steps.run-date.outputs.value }} steps: - name: checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: set up pnpm - uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - cache: true - require-lockfile: true + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: set run date + id: run-date + run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT" + - name: create experiment plan + env: + RUN_DATE: ${{ steps.run-date.outputs.value }} + run: script/run-experiment.sh "${{ inputs.experiment }}" plan + - name: upload experiment plan + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: experiment-plan-${{ github.run_id }} + path: results + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + run: + needs: plan + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + shard: [1, 2, 3, 4] + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - name: set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile - name: build project run: pnpm run build - - name: run experiment + - name: download experiment plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: experiment-plan-${{ github.run_id }} + - name: run experiment shard env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} CONCURRENCY: ${{ inputs.concurrency }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - run: script/run-experiment.sh "${{ inputs.experiment }}" - - name: prepare experiment artifact + RUN_DATE: ${{ needs.plan.outputs.run-date }} + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + run: script/run-experiment.sh "${{ inputs.experiment }}" shard + - name: prepare experiment shard artifact if: ${{ always() }} run: | mkdir -p workflow-artifact if [[ -d results ]]; then mv results workflow-artifact/results fi - - name: upload experiment results + - name: upload experiment shard if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: experiment-shard-${{ github.run_id }}-${{ matrix.shard }} + path: workflow-artifact + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + merge: + needs: [plan, run] + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 + with: + cache: true + require-lockfile: true + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + - name: build project + run: pnpm run build + - name: download experiment shards + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: experiment-shard-${{ github.run_id }}-* + merge-multiple: true + - name: merge experiment shards + env: + RUN_DATE: ${{ needs.plan.outputs.run-date }} + run: script/run-experiment.sh "${{ inputs.experiment }}" merge + - name: prepare experiment artifact + run: | + mkdir -p workflow-artifact + mv results workflow-artifact/results + - name: upload experiment results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: experiment-${{ github.run_id }} diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 528ab803..108adac9 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -164,6 +164,42 @@ directory to preserve those references. `--output-dir` creates `output.json` and `artifacts/` within the selected directory. When using `--output`, artifacts are written to an `artifacts/` directory beside the selected file. +### Plans and sharding + +Create a durable, randomized trial plan before running an experiment or +benchmark: + +```sh +agent-eval --experiment example --plan plan.json +``` + +Plan creation does not require a Copilot token. The plan stores the ordered +trial IDs and references needed to reload the experiment or benchmark. Keep the +same experiment, benchmark, and scenario configuration available when running +the plan. + +Run deterministic shards from the shared plan, writing a distinct output file +for each shard: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval \ + --from-plan plan.json \ + --shard 1/4 \ + --artifacts run/artifacts \ + --output run/output-1.json +``` + +After all shards finish, merge the `output-*.json` files into one portable +result: + +```sh +agent-eval --merge-shards run --output run/output.json +``` + +`--plan` and `--from-plan` default to `plan.json` when their path is omitted. +`--shard` is only valid with `--from-plan`. Shard merging does not require a +Copilot token. + ## Scenario config authoring Use `defineConfig` from `@primer/agent-eval/scenario` in each diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index 0344e8a5..d06e56d7 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -18,6 +18,10 @@ "types": "./dist/experiment.d.ts", "default": "./dist/experiment.js" }, + "./plan": { + "types": "./dist/plan.d.ts", + "default": "./dist/plan.js" + }, "./scenario": { "types": "./dist/scenario.d.ts", "default": "./dist/scenario.js" diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index 7e1d288b..dfae1a90 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -13,6 +13,7 @@ const config = defineConfig({ cli: 'src/cli.ts', experiment: 'src/experiment.ts', index: 'src/index.ts', + plan: 'src/plan.ts', scenario: 'src/scenario.ts', }, platform: 'node', diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts index 314a75e5..67506b3d 100644 --- a/packages/agent-eval/src/benchmark.test.ts +++ b/packages/agent-eval/src/benchmark.test.ts @@ -1,14 +1,19 @@ import {afterEach, expect, test, vi} from 'vitest' import { + createPlan, defineConfig, getBenchmark, listBenchmarks, + merge, output, read, + resolvePlan, run, write, + type Benchmark, type BenchmarkTrialResult, } from './benchmark' +import {deserialize as deserializePlan, isBenchmarkPlan} from './plan' import {VirtualHost} from './host' import {run as runPlan} from './plan' import {defineConfig as defineScenarioConfig} from './scenario' @@ -440,4 +445,105 @@ test('writes and reads benchmark capability metadata', async () => { 'utf-8', ) await expect(read('/bundle/embedded-output.json', {host})).rejects.toThrow() + + const secondOutput = output('test-benchmark', [ + { + ...trialResult, + trial: { + ...trialResult.trial, + id: 'trial-two', + }, + }, + ]) + const merged = merge([benchmarkOutput, secondOutput]) + + expect(merged.capabilities.get('Test capability')).toEqual({ + name: 'Test capability', + scenarioIds: ['001-scenario'], + }) + expect([...merged.trials.keys()]).toEqual(['trial', 'trial-two']) + expect(() => { + merge([benchmarkOutput, benchmarkOutput]) + }).toThrow('Cannot merge duplicate trial id: trial') +}) + +test('resolves durable benchmark plans with capability metadata and setup functions', () => { + const setup = async () => { + return undefined + } + const benchmark: Benchmark = { + id: 'test-benchmark', + filepath: '/benchmarks/test-benchmark.ts', + name: 'Test benchmark', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + setup, + capabilities: [ + { + name: 'Test capability', + scenarios: [ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }, + ], + }, + ], + } + + const plan = createPlan(benchmark) + const resolved = resolvePlan(benchmark, plan) + const benchmarkTrial = resolved.plan.trials.find(trial => trial.treatment.name === 'Benchmark') + + expect(resolved.plan.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(benchmarkTrial?.treatment.setup).toBe(setup) + expect(resolved.trialCapabilities.get(benchmarkTrial?.id ?? '')).toBe(benchmark.capabilities[0]) +}) + +test('rejects benchmark plans with references missing from the current config', () => { + const benchmark: Benchmark = { + id: 'test-benchmark', + filepath: '/benchmarks/test-benchmark.ts', + name: 'Test benchmark', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + capabilities: [], + } + const plan = deserializePlan({ + version: 1, + source: { + kind: 'benchmark', + id: benchmark.id, + }, + trials: [ + { + id: 'trial', + scenarioId: '001-scenario', + treatmentId: 'Benchmark', + model: benchmark.models[0], + capabilityId: 'Missing capability', + }, + ], + }) + + if (!isBenchmarkPlan(plan)) { + throw new Error('Expected benchmark plan') + } + + expect(() => { + resolvePlan(benchmark, plan) + }).toThrow('Plan trial "trial" references missing benchmark capability: Missing capability') }) diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts index a8287a3b..4e2681f2 100644 --- a/packages/agent-eval/src/benchmark.ts +++ b/packages/agent-eval/src/benchmark.ts @@ -5,7 +5,13 @@ import type {EnvironmentConfig} from './environment' import {DefaultHost, type Host} from './host' import {logger} from './logger' import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model' -import {create as createPlan, run as runPlan} from './plan' +import { + create as createDurablePlan, + run as runPlan, + type BenchmarkPlan, + type BenchmarkPlanTrialReference, + type RuntimePlan, +} from './plan' import {getScenario, ScenarioSchema, type Scenario} from './scenario' import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' import { @@ -174,48 +180,153 @@ type BenchmarkTrialResult = TrialResult & { type BenchmarkRunResult = Array +function createPlan(benchmark: Benchmark): BenchmarkPlan { + const capabilityIds = new Set() + + for (const capability of benchmark.capabilities) { + if (capabilityIds.has(capability.name)) { + throw new Error(`Benchmark "${benchmark.id}" contains duplicate capability id: ${capability.name}`) + } + + capabilityIds.add(capability.name) + } + + const trials: Array = benchmark.models.flatMap(model => { + return benchmark.capabilities.flatMap(capability => { + return capability.scenarios.flatMap(scenario => { + return [ControlTreatment.name, 'Benchmark'].map(treatmentId => { + return { + id: randomUUID(), + scenarioId: scenario.id, + treatmentId, + model, + capabilityId: capability.name, + } + }) + }) + }) + }) + + return createDurablePlan({ + source: { + kind: 'benchmark', + id: benchmark.id, + }, + trials, + }) +} + +function resolvePlan( + benchmark: Benchmark, + plan: BenchmarkPlan, +): { + plan: RuntimePlan + trialCapabilities: Map +} { + if (plan.source.kind !== 'benchmark') { + throw new Error(`Expected a benchmark plan, received: ${plan.source.kind}`) + } + + if (plan.source.id !== benchmark.id) { + throw new Error(`Plan references benchmark "${plan.source.id}", but loaded benchmark "${benchmark.id}"`) + } + + const capabilities = new Map() + for (const capability of benchmark.capabilities) { + if (capabilities.has(capability.name)) { + throw new Error(`Benchmark "${benchmark.id}" contains duplicate capability id: ${capability.name}`) + } + + capabilities.set(capability.name, capability) + } + const trialCapabilities = new Map() + const trials: Array = plan.trials.map(reference => { + const capability = capabilities.get(reference.capabilityId) + if (!capability) { + throw new Error(`Plan trial "${reference.id}" references missing benchmark capability: ${reference.capabilityId}`) + } + + const scenario = capability.scenarios.find(candidate => candidate.id === reference.scenarioId) + if (!scenario) { + throw new Error( + `Plan trial "${reference.id}" references scenario "${reference.scenarioId}" outside capability "${reference.capabilityId}"`, + ) + } + + const model = benchmark.models.find(candidate => { + return candidate.name === reference.model.name && candidate.reasoningEffort === reference.model.reasoningEffort + }) + if (!model) { + throw new Error( + `Plan trial "${reference.id}" references missing model variant: ${reference.model.name}/${reference.model.reasoningEffort}`, + ) + } + + let treatment + if (reference.treatmentId === ControlTreatment.name) { + treatment = ControlTreatment + } else if (reference.treatmentId === 'Benchmark') { + treatment = createBenchmarkTreatment(benchmark, capability) + } else { + throw new Error(`Plan trial "${reference.id}" references missing benchmark treatment: ${reference.treatmentId}`) + } + + trialCapabilities.set(reference.id, capability) + return { + id: reference.id, + scenario, + treatment, + model, + } + }) + + return { + plan: { + trials, + }, + trialCapabilities, + } +} + async function run({ env, host = DefaultHost, id, + plan, }: { env: EnvironmentConfig host?: Host - id: string + id?: string + plan?: BenchmarkPlan }): Promise { + if (id && plan) { + throw new Error('Benchmark run accepts either an id or a plan, not both') + } + + if (!id && !plan) { + throw new Error('Benchmark run requires an id or a plan') + } + + const benchmarkId = plan?.source.id ?? id + if (!benchmarkId) { + throw new Error('Benchmark run requires an id or a plan') + } + const benchmark = await getBenchmark({ host, benchmarksDirectory: env.benchmarksDirectory, scenariosDirectory: env.scenariosDirectory, - id, + id: benchmarkId, }) - const trialCapabilities = new Map() - const trials: Array = benchmark.models.flatMap(model => { - return benchmark.capabilities.flatMap(capability => { - const benchmarkTreatment = createBenchmarkTreatment(benchmark, capability) - return capability.scenarios.flatMap(scenario => { - return [ControlTreatment, benchmarkTreatment].map(treatment => { - const trial = { - id: randomUUID(), - scenario, - treatment, - model, - } - trialCapabilities.set(trial.id, capability) - return trial - }) - }) - }) - }) - const plan = await createPlan(trials) + const resolved = resolvePlan(benchmark, plan ?? createPlan(benchmark)) const results = await runPlan({ env, host, - plan, + plan: resolved.plan, }) return results.map(result => { - const capability = trialCapabilities.get(result.trial.id) + const capability = resolved.trialCapabilities.get(result.trial.id) if (!capability) { throw new Error(`Capability was not found for trial: ${result.trial.id}`) } @@ -386,7 +497,67 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } -export {BenchmarkConfigSchema, defineConfig, getBenchmark, listBenchmarks, output, read, run, write} +function merge(outputs: Array): BenchmarkOutput { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one benchmark output is required to merge shards') + } + + const result: BenchmarkOutput = { + benchmarkId: first.benchmarkId, + capabilities: new Map(first.capabilities), + scenarios: new Map(first.scenarios), + treatments: new Map(first.treatments), + trials: new Map(first.trials), + } + + for (const shardOutput of remaining) { + if (shardOutput.benchmarkId !== result.benchmarkId) { + throw new Error( + `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${shardOutput.benchmarkId}"`, + ) + } + + mergeMetadataMap(result.capabilities, shardOutput.capabilities, 'capability') + mergeMetadataMap(result.scenarios, shardOutput.scenarios, 'scenario') + mergeMetadataMap(result.treatments, shardOutput.treatments, 'treatment') + + for (const [trialId, trial] of shardOutput.trials) { + if (result.trials.has(trialId)) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } + + result.trials.set(trialId, trial) + } + } + + return result +} + +function mergeMetadataMap(target: Map, source: Map, type: string): void { + for (const [id, value] of source) { + const existing = target.get(id) + if (target.has(id) && JSON.stringify(existing) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target.set(id, value) + } +} + +export { + BenchmarkConfigSchema, + createPlan, + defineConfig, + getBenchmark, + listBenchmarks, + merge, + output, + read, + resolvePlan, + run, + write, +} export type { BenchmarkConfig, Benchmark, diff --git a/packages/agent-eval/src/cli-options.test.ts b/packages/agent-eval/src/cli-options.test.ts new file mode 100644 index 00000000..54b93433 --- /dev/null +++ b/packages/agent-eval/src/cli-options.test.ts @@ -0,0 +1,77 @@ +import {describe, expect, test} from 'vitest' +import {getCliMode, normalizeOptionalPathArguments} from './cli-options' + +describe('normalizeOptionalPathArguments', () => { + test('adds defaults for bare optional path flags', () => { + expect(normalizeOptionalPathArguments(['--benchmark', 'test', '--plan'])).toEqual([ + '--benchmark', + 'test', + '--plan=plan.json', + ]) + expect(normalizeOptionalPathArguments(['--from-plan', '--shard', '2/3'])).toEqual([ + '--from-plan=plan.json', + '--shard', + '2/3', + ]) + expect(normalizeOptionalPathArguments(['--merge-shards'])).toEqual(['--merge-shards=']) + }) + + test('preserves explicit optional paths', () => { + expect(normalizeOptionalPathArguments(['--plan', 'plans/test.json'])).toEqual(['--plan', 'plans/test.json']) + }) +}) + +describe('getCliMode', () => { + test('creates plan and from-plan modes', () => { + expect( + getCliMode({ + benchmark: 'test', + plan: 'plan.json', + }), + ).toEqual({ + kind: 'create-plan', + sourceKind: 'benchmark', + sourceId: 'test', + path: 'plan.json', + }) + expect( + getCliMode({ + 'from-plan': 'plan.json', + shard: '2/3', + }), + ).toEqual({ + kind: 'from-plan', + path: 'plan.json', + shard: '2/3', + }) + }) + + test('validates incompatible modes', () => { + expect(() => { + getCliMode({ + benchmark: 'benchmark', + experiment: 'experiment', + }) + }).toThrow('--benchmark and --experiment cannot be combined') + + expect(() => { + getCliMode({ + benchmark: 'benchmark', + 'from-plan': 'plan.json', + }) + }).toThrow('--from-plan cannot be combined') + + expect(() => { + getCliMode({ + benchmark: 'benchmark', + shard: '1/2', + }) + }).toThrow('--shard is only valid with --from-plan') + + expect(() => { + getCliMode({ + plan: 'plan.json', + }) + }).toThrow('--plan requires --benchmark or --experiment') + }) +}) diff --git a/packages/agent-eval/src/cli-options.ts b/packages/agent-eval/src/cli-options.ts new file mode 100644 index 00000000..2e280df7 --- /dev/null +++ b/packages/agent-eval/src/cli-options.ts @@ -0,0 +1,149 @@ +const DEFAULT_PLAN_PATH = 'plan.json' + +const optionalPathDefaults = new Map([ + ['--plan', DEFAULT_PLAN_PATH], + ['--from-plan', DEFAULT_PLAN_PATH], + ['--merge-shards', ''], +]) + +type CliModeOptions = { + benchmark?: string + experiment?: string + plan?: string + 'from-plan'?: string + 'merge-shards'?: string + shard?: string +} + +type CliMode = + | { + kind: 'none' + } + | { + kind: 'benchmark' + id: string + } + | { + kind: 'experiment' + id: string + } + | { + kind: 'create-plan' + sourceKind: 'benchmark' | 'experiment' + sourceId: string + path: string + } + | { + kind: 'from-plan' + path: string + shard?: string + } + | { + kind: 'merge-shards' + directory?: string + } + +function normalizeOptionalPathArguments(args: Array): Array { + const normalized: Array = [] + + for (let index = 0; index < args.length; index++) { + const argument = args[index] + const defaultValue = optionalPathDefaults.get(argument) + if (defaultValue === undefined) { + normalized.push(argument) + continue + } + + const next = args[index + 1] + if (next === undefined || next.startsWith('-')) { + normalized.push(`${argument}=${defaultValue}`) + continue + } + + normalized.push(argument, next) + index += 1 + } + + return normalized +} + +function getCliMode(options: CliModeOptions): CliMode { + if (options.benchmark && options.experiment) { + throw new Error('--benchmark and --experiment cannot be combined') + } + + if (options['from-plan']) { + if ( + options.benchmark || + options.experiment || + options.plan !== undefined || + options['merge-shards'] !== undefined + ) { + throw new Error('--from-plan cannot be combined with --benchmark, --experiment, --plan, or --merge-shards') + } + + return { + kind: 'from-plan', + path: options['from-plan'], + shard: options.shard, + } + } + + if (options['merge-shards'] !== undefined) { + if (options.benchmark || options.experiment || options.plan !== undefined) { + throw new Error('--merge-shards cannot be combined with --benchmark, --experiment, or --plan') + } + + if (options.shard) { + throw new Error('--shard is only valid with --from-plan') + } + + return { + kind: 'merge-shards', + directory: options['merge-shards'] || undefined, + } + } + + if (options.shard) { + throw new Error('--shard is only valid with --from-plan') + } + + if (options.plan !== undefined) { + if (!options.benchmark && !options.experiment) { + throw new Error('--plan requires --benchmark or --experiment') + } + + const sourceId = options.benchmark ?? options.experiment + if (!sourceId) { + throw new Error('--plan requires --benchmark or --experiment') + } + + return { + kind: 'create-plan', + sourceKind: options.benchmark ? 'benchmark' : 'experiment', + sourceId, + path: options.plan, + } + } + + if (options.benchmark) { + return { + kind: 'benchmark', + id: options.benchmark, + } + } + + if (options.experiment) { + return { + kind: 'experiment', + id: options.experiment, + } + } + + return { + kind: 'none', + } +} + +export {DEFAULT_PLAN_PATH, getCliMode, normalizeOptionalPathArguments} +export type {CliMode, CliModeOptions} diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts index 7f42fdba..2a2579a6 100644 --- a/packages/agent-eval/src/cli.test.ts +++ b/packages/agent-eval/src/cli.test.ts @@ -25,10 +25,11 @@ describe('cli', () => { await expect(import('./cli')).rejects.toThrow('process.exit') expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]')) expect(log).toHaveBeenCalledWith(expect.stringContaining('--output-dir ')) + expect(log).toHaveBeenCalledWith(expect.stringContaining('--from-plan [path]')) }) - test('requires a Copilot token before running', async () => { - process.argv = ['node', 'agent-eval'] + test('requires a Copilot token when running', async () => { + process.argv = ['node', 'agent-eval', '--benchmark', 'test'] delete process.env.COPILOT_GITHUB_TOKEN await expect(import('./cli')).rejects.toThrow( @@ -38,7 +39,7 @@ describe('cli', () => { test('displays help when no benchmark or experiment is selected', async () => { process.argv = ['node', 'agent-eval'] - process.env.COPILOT_GITHUB_TOKEN = 'token' + delete process.env.COPILOT_GITHUB_TOKEN const log = vi.spyOn(console, 'log').mockImplementation(() => {}) await import('./cli') @@ -52,4 +53,18 @@ describe('cli', () => { await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --output') }) + + test('does not require a Copilot token when creating a plan', async () => { + process.argv = ['node', 'agent-eval', '--benchmark', 'test', '--plan'] + delete process.env.COPILOT_GITHUB_TOKEN + + await expect(import('./cli')).rejects.toThrow('Benchmark "test" was not found') + }) + + test('does not require a Copilot token when merging shard outputs', async () => { + process.argv = ['node', 'agent-eval', '--merge-shards', '/missing-agent-eval-shards'] + delete process.env.COPILOT_GITHUB_TOKEN + + await expect(import('./cli')).rejects.toThrow('ENOENT') + }) }) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index 95b2b072..ac8d1ecf 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -3,25 +3,39 @@ import fs from 'node:fs/promises' import path from 'node:path' import {parseArgs} from 'node:util' +import {getCliMode, normalizeOptionalPathArguments} from './cli-options' import {getEnvironmentConfig} from './environment' import { + createPlan as createBenchmarkPlan, getBenchmark, run as runBenchmark, output as getBenchmarkOutput, write as writeBenchmarkOutput, } from './benchmark' import { + createPlan as createExperimentPlan, getExperiment, run as runExperiment, output as getExperimentOutput, write as writeExperimentOutput, } from './experiment' import {logger} from './logger' +import {mergeShardOutputs} from './output' +import { + deserialize as deserializePlan, + isBenchmarkPlan, + select as selectPlan, + serialize as serializePlan, + type BenchmarkPlan, + type ExperimentPlan, + type Plan, +} from './plan' import {formatBenchmarkResults, formatExperimentResults} from './report' import {parseShard} from './shard' import {compare as compareTrial} from './trial' const {values} = parseArgs({ + args: normalizeOptionalPathArguments(process.argv.slice(2)), options: { benchmark: { type: 'string', @@ -72,9 +86,21 @@ const {values} = parseArgs({ type: 'string', description: 'The directory containing scenario directories', }, + plan: { + type: 'string', + description: 'Create a durable plan without running it', + }, + 'from-plan': { + type: 'string', + description: 'Run trials from a durable plan', + }, + 'merge-shards': { + type: 'string', + description: 'Merge output-*.json shard outputs from a directory', + }, shard: { type: 'string', - description: 'The experiment shard to run, formatted as order/total', + description: 'The durable plan shard to run, formatted as order/total', }, }, }) @@ -94,8 +120,11 @@ Options: --log-level The log level to use (default: info) --output The target file in which results are written (default: output.json) --output-dir The directory containing output.json and its artifacts + --plan [path] Create a durable plan without running it (default: plan.json) + --from-plan [path] Run trials from a durable plan (default: plan.json) + --merge-shards [dir] Merge output-*.json files (default: output file directory) --scenarios The directory containing scenario directories (default: ./scenarios) - --shard The experiment shard to run + --shard Select a deterministic shard from --from-plan `) } @@ -110,15 +139,12 @@ if (values['log-level']) { const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY - -if (!COPILOT_GITHUB_TOKEN) { - throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval') -} +const mode = getCliMode(values) const env = getEnvironmentConfig({ benchmarksDirectory: values.benchmarks, concurrency: values.concurrency, - copilotToken: COPILOT_GITHUB_TOKEN, + copilotToken: COPILOT_GITHUB_TOKEN ?? '', dockerImage: values['docker-image']?.trim(), experimentsDirectory: values.experiments, outputDirectory: values['output-dir'], @@ -128,17 +154,68 @@ const env = getEnvironmentConfig({ logger.debug('Environment configuration: %o', env) -if (values.benchmark) { - logger.info('Running benchmark: %s', values.benchmark) +if (mode.kind === 'create-plan') { + const planPath = path.resolve(mode.path) + let plan: Plan + + if (mode.sourceKind === 'benchmark') { + const benchmark = await getBenchmark({ + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id: mode.sourceId, + }) + plan = createBenchmarkPlan(benchmark) + } else { + const experiment = await getExperiment({ + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id: mode.sourceId, + }) + plan = createExperimentPlan(experiment) + } + + await ensureParentDirectory(planPath) + logger.info('Writing plan to: %s', planPath) + await fs.writeFile(planPath, serializePlan(plan), 'utf-8') +} else if (mode.kind === 'merge-shards') { + const directory = path.resolve(mode.directory ?? path.dirname(env.outputPath)) + const entries = await fs.readdir(directory, { + withFileTypes: true, + }) + const filenames = entries + .filter(entry => { + return entry.isFile() && /^output-.*\.json$/.test(entry.name) + }) + .map(entry => { + return entry.name + }) + .toSorted() + const inputs = filenames.map(filename => { + return path.join(directory, filename) + }) + const merged = await mergeShardOutputs(inputs, { + targetDirectory: path.dirname(env.outputPath), + }) + + await ensureParentDirectory(env.outputPath) + logger.info('Writing merged %s output to: %s', merged.kind, env.outputPath) + if (merged.kind === 'benchmark') { + await writeBenchmarkOutput(env.outputPath, merged.output) + } else { + await writeExperimentOutput(env.outputPath, merged.output) + } +} else if (mode.kind === 'benchmark') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + logger.info('Running benchmark: %s', mode.id) const benchmark = await getBenchmark({ benchmarksDirectory: env.benchmarksDirectory, scenariosDirectory: env.scenariosDirectory, - id: values.benchmark, + id: mode.id, }) const result = await runBenchmark({ env, - id: values.benchmark, + id: mode.id, }) const sorted = result.toSorted(compareTrial) @@ -156,18 +233,18 @@ if (values.benchmark) { if (GITHUB_STEP_SUMMARY) { await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } -} else if (values.experiment) { - logger.info('Running experiment: %s', values.experiment) +} else if (mode.kind === 'experiment') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + logger.info('Running experiment: %s', mode.id) const experiment = await getExperiment({ experimentsDirectory: env.experimentsDirectory, scenariosDirectory: env.scenariosDirectory, - id: values.experiment, + id: mode.id, }) const result = await runExperiment({ env, - id: values.experiment, - shard: values.shard ? parseShard(values.shard) : undefined, + id: mode.id, }) const sorted = result.toSorted(compareTrial) @@ -185,6 +262,97 @@ if (values.benchmark) { if (GITHUB_STEP_SUMMARY) { await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } +} else if (mode.kind === 'from-plan') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + const planPath = path.resolve(mode.path) + const durablePlan = deserializePlan(await fs.readFile(planPath, 'utf-8')) + const plan = mode.shard ? selectDurablePlan(durablePlan, mode.shard) : durablePlan + + if (isBenchmarkPlan(plan)) { + await runBenchmarkFromPlan(plan) + } else { + await runExperimentFromPlan(plan) + } } else { displayHelp() } + +function requireCopilotToken(token: string | undefined): asserts token is string { + if (!token) { + throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval') + } +} + +function selectDurablePlan(plan: Plan, shardValue: string): Plan { + const shard = parseShard(shardValue) + if (isBenchmarkPlan(plan)) { + return selectPlan(plan, shard) + } + + return selectPlan(plan, shard) +} + +async function ensureParentDirectory(filepath: string): Promise { + const directory = path.dirname(filepath) + if (!existsSync(directory)) { + await fs.mkdir(directory, {recursive: true}) + } +} + +async function runBenchmarkFromPlan(plan: BenchmarkPlan): Promise { + logger.info('Running benchmark plan: %s', plan.source.id) + + const benchmark = await getBenchmark({ + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id: plan.source.id, + }) + const result = await runBenchmark({ + env, + plan, + }) + const sorted = result.toSorted(compareTrial) + + await ensureParentDirectory(env.outputPath) + await writeBenchmarkOutput( + env.outputPath, + getBenchmarkOutput(benchmark.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }), + ) + + const resultSummaries = formatBenchmarkResults(benchmark, sorted) + console.log(resultSummaries) + + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) + } +} + +async function runExperimentFromPlan(plan: ExperimentPlan): Promise { + logger.info('Running experiment plan: %s', plan.source.id) + + const experiment = await getExperiment({ + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id: plan.source.id, + }) + const result = await runExperiment({ + env, + plan, + }) + const sorted = result.toSorted(compareTrial) + const output = getExperimentOutput(experiment.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }) + + await ensureParentDirectory(env.outputPath) + await writeExperimentOutput(env.outputPath, output) + + const resultSummaries = formatExperimentResults(experiment.name, sorted) + console.log(resultSummaries) + + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) + } +} diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts index b71c42c6..0d349161 100644 --- a/packages/agent-eval/src/experiment.test.ts +++ b/packages/agent-eval/src/experiment.test.ts @@ -1,8 +1,20 @@ import path from 'node:path' import {expect, test} from 'vitest' -import {defineConfig, getExperiment, listExperiments, output, read, write} from './experiment' +import { + createPlan, + defineConfig, + getExperiment, + listExperiments, + merge, + output, + read, + resolvePlan, + write, + type Experiment, +} from './experiment' import {VirtualHost} from './host' import type {TrialResult} from './trial' +import {deserialize as deserializePlan, isBenchmarkPlan} from './plan' const config = defineConfig({ name: 'Test experiment', @@ -404,4 +416,166 @@ test('creates portable artifact paths relative to the output directory', async ( await expect(write('/bundle/output.json', portableOutput, {host})).rejects.toThrow( 'must not resolve outside the output directory', ) + + const secondOutput = output('baseline', [ + { + ...trialResult, + trial: { + ...trialResult.trial, + id: 'trial-two', + }, + }, + ]) + const merged = merge([portableOutput, secondOutput]) + + expect([...merged.trials.keys()]).toEqual(['trial', 'trial-two']) + expect(() => { + merge([portableOutput, portableOutput]) + }).toThrow('Cannot merge duplicate trial id: trial') + expect(() => { + merge([portableOutput, output('different', [])]) + }).toThrow('Cannot merge experiment outputs for different sources') +}) + +test('resolves durable experiment plans with experiment and treatment setup functions', () => { + const experimentSetup = async () => { + return undefined + } + const treatmentSetup = async () => { + return undefined + } + const experiment: Experiment = { + id: 'test-experiment', + filepath: '/experiments/test-experiment.ts', + name: 'Test experiment', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + scenarios: [ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }, + ], + setup: experimentSetup, + treatments: [ + { + name: 'Treatment', + setup: treatmentSetup, + }, + ], + } + + const plan = createPlan(experiment) + const resolved = resolvePlan(experiment, plan) + const treatmentTrial = resolved.trials.find(trial => trial.treatment.name === 'Treatment') + + expect(resolved.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(treatmentTrial?.setup).toBe(experimentSetup) + expect(treatmentTrial?.treatment.setup).toBe(treatmentSetup) +}) + +test('rejects experiment plans with references missing from the current config', () => { + const scenario = { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + } + const experiment: Experiment = { + id: 'test-experiment', + filepath: '/experiments/test-experiment.ts', + name: 'Test experiment', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + scenarios: [scenario], + treatments: [], + } + const missingScenarioPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: 'missing-scenario', + treatmentId: 'Control', + model: experiment.models[0], + }, + ], + }) + + if (isBenchmarkPlan(missingScenarioPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingScenarioPlan) + }).toThrow('Plan trial "trial" references missing scenario: missing-scenario') + + const missingTreatmentPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: scenario.id, + treatmentId: 'Missing treatment', + model: experiment.models[0], + }, + ], + }) + + if (isBenchmarkPlan(missingTreatmentPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingTreatmentPlan) + }).toThrow('Plan trial "trial" references missing treatment: Missing treatment') + + const missingModelPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: scenario.id, + treatmentId: 'Control', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'high', + }, + }, + ], + }) + + if (isBenchmarkPlan(missingModelPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingModelPlan) + }).toThrow('Plan trial "trial" references missing model variant: gpt-5.6-sol/high') }) diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts index 5d36a84a..fb5bfb00 100644 --- a/packages/agent-eval/src/experiment.ts +++ b/packages/agent-eval/src/experiment.ts @@ -11,7 +11,13 @@ import { } from './model' import {DefaultHost, type Host} from './host' import {logger} from './logger' -import {create as createPlan, run as runPlan} from './plan' +import { + create as createDurablePlan, + run as runPlan, + type ExperimentPlan, + type ExperimentPlanTrialReference, + type RuntimePlan, +} from './plan' import {getScenario, loadScenario, ScenarioSchema, type Scenario} from './scenario' import {selectShard, type Shard} from './shard' import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' @@ -23,7 +29,6 @@ import { WalkthroughSchema, writeTrialFiles, type ResultFileOptions, - type Trial, type TrialResult, } from './trial' import {TestResultsSchema} from './vitest' @@ -179,50 +184,132 @@ async function getExperiment({ type ExperimentRunResult = Array +function createPlan(experiment: Experiment): ExperimentPlan { + const treatmentIds = new Set([ControlTreatment.name]) + + for (const treatment of experiment.treatments) { + if (treatmentIds.has(treatment.name)) { + throw new Error(`Experiment "${experiment.id}" contains duplicate treatment id: ${treatment.name}`) + } + + treatmentIds.add(treatment.name) + } + + const trials: Array = experiment.models.flatMap(model => { + return experiment.scenarios.flatMap(scenario => { + return [ControlTreatment, ...experiment.treatments].map(treatment => { + return { + id: randomUUID(), + scenarioId: scenario.id, + treatmentId: treatment.name, + model, + } + }) + }) + }) + + return createDurablePlan({ + source: { + kind: 'experiment', + id: experiment.id, + }, + trials, + }) +} + +function resolvePlan(experiment: Experiment, plan: ExperimentPlan): RuntimePlan { + if (plan.source.kind !== 'experiment') { + throw new Error(`Expected an experiment plan, received: ${plan.source.kind}`) + } + + if (plan.source.id !== experiment.id) { + throw new Error(`Plan references experiment "${plan.source.id}", but loaded experiment "${experiment.id}"`) + } + + const treatments = new Map([[ControlTreatment.name, ControlTreatment]]) + for (const treatment of experiment.treatments) { + if (treatments.has(treatment.name)) { + throw new Error(`Experiment "${experiment.id}" contains duplicate treatment id: ${treatment.name}`) + } + + treatments.set(treatment.name, treatment) + } + + return { + trials: plan.trials.map(reference => { + const scenario = experiment.scenarios.find(candidate => candidate.id === reference.scenarioId) + if (!scenario) { + throw new Error(`Plan trial "${reference.id}" references missing scenario: ${reference.scenarioId}`) + } + + const treatment = treatments.get(reference.treatmentId) + if (!treatment) { + throw new Error(`Plan trial "${reference.id}" references missing treatment: ${reference.treatmentId}`) + } + + const model = experiment.models.find(candidate => { + return candidate.name === reference.model.name && candidate.reasoningEffort === reference.model.reasoningEffort + }) + if (!model) { + throw new Error( + `Plan trial "${reference.id}" references missing model variant: ${reference.model.name}/${reference.model.reasoningEffort}`, + ) + } + + return { + id: reference.id, + scenario, + treatment, + model, + setup: experiment.setup, + } + }), + } +} + async function run({ env, host = DefaultHost, id, + plan, shard, }: { env: EnvironmentConfig host?: Host - id: string + id?: string + plan?: ExperimentPlan shard?: Shard }): Promise { + if (id && plan) { + throw new Error('Experiment run accepts either an id or a plan, not both') + } + + if (!id && !plan) { + throw new Error('Experiment run requires an id or a plan') + } + + const experimentId = plan?.source.id ?? id + if (!experimentId) { + throw new Error('Experiment run requires an id or a plan') + } + const experiment = await getExperiment({ host, experimentsDirectory: env.experimentsDirectory, scenariosDirectory: env.scenariosDirectory, - id, - }) - const trials: Array = experiment.models.flatMap(model => { - return experiment.scenarios.flatMap(scenario => { - return [ - { - id: randomUUID(), - scenario, - treatment: ControlTreatment, - model, - setup: experiment.setup, - }, - ...experiment.treatments.map(treatment => { - return { - id: randomUUID(), - scenario, - treatment, - model, - setup: experiment.setup, - } - }), - ] - }) + id: experimentId, }) - const plan = await createPlan(shard ? selectShard(trials, shard) : trials) + const durablePlan = plan ?? createPlan(experiment) + const selectedPlan = shard + ? { + ...durablePlan, + trials: selectShard(durablePlan.trials, shard), + } + : durablePlan const results = await runPlan({ env, host, - plan, + plan: resolvePlan(experiment, selectedPlan), }) return results @@ -351,7 +438,65 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } -export {ExperimentConfigSchema, defineConfig, getExperiment, listExperiments, output, read, run, write} +function merge(outputs: Array): ExperimentOutput { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one experiment output is required to merge shards') + } + + const result: ExperimentOutput = { + experimentId: first.experimentId, + scenarios: new Map(first.scenarios), + treatments: new Map(first.treatments), + trials: new Map(first.trials), + } + + for (const shardOutput of remaining) { + if (shardOutput.experimentId !== result.experimentId) { + throw new Error( + `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${shardOutput.experimentId}"`, + ) + } + + mergeMetadataMap(result.scenarios, shardOutput.scenarios, 'scenario') + mergeMetadataMap(result.treatments, shardOutput.treatments, 'treatment') + + for (const [trialId, trial] of shardOutput.trials) { + if (result.trials.has(trialId)) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } + + result.trials.set(trialId, trial) + } + } + + return result +} + +function mergeMetadataMap(target: Map, source: Map, type: string): void { + for (const [id, value] of source) { + const existing = target.get(id) + if (target.has(id) && JSON.stringify(existing) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target.set(id, value) + } +} + +export { + ExperimentConfigSchema, + createPlan, + defineConfig, + getExperiment, + listExperiments, + merge, + output, + read, + resolvePlan, + run, + write, +} export type { ExperimentConfig, Experiment, diff --git a/packages/agent-eval/src/index.test.ts b/packages/agent-eval/src/index.test.ts index c74c9077..24cc47ca 100644 --- a/packages/agent-eval/src/index.test.ts +++ b/packages/agent-eval/src/index.test.ts @@ -3,6 +3,7 @@ import { BenchmarkConfigSchema, ControlTreatment, ExperimentConfigSchema, + PlanSchema, ScenarioConfigSchema, TreatmentSchema, TrialResultSchema, @@ -10,6 +11,8 @@ import { defineBenchmarkConfig, defineExperimentConfig, defineScenarioConfig, + deserializePlan, + serializePlan, } from './index' test('exports the public configuration helpers and schemas', () => { @@ -22,4 +25,7 @@ test('exports the public configuration helpers and schemas', () => { expect(TreatmentSchema.parse(ControlTreatment)).toEqual(ControlTreatment) expect(TrialSchema).toBeDefined() expect(TrialResultSchema).toBeDefined() + expect(PlanSchema).toBeDefined() + expect(deserializePlan).toBeTypeOf('function') + expect(serializePlan).toBeTypeOf('function') }) diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index 1b63f017..4ca40efa 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -1,10 +1,13 @@ export { BenchmarkConfigSchema, + createPlan as createBenchmarkPlan, defineConfig as defineBenchmarkConfig, getBenchmark, listBenchmarks, + merge as mergeBenchmarkOutputs, output as getBenchmarkOutput, read as readBenchmarkOutput, + resolvePlan as resolveBenchmarkPlan, run as runBenchmark, write as writeBenchmarkOutput, } from './benchmark' @@ -20,11 +23,14 @@ export type { export { ExperimentConfigSchema, + createPlan as createExperimentPlan, defineConfig as defineExperimentConfig, getExperiment, listExperiments, + merge as mergeExperimentOutputs, output as getExperimentOutput, read as readExperimentOutput, + resolvePlan as resolveExperimentPlan, run as runExperiment, write as writeExperimentOutput, } from './experiment' @@ -38,3 +44,25 @@ export type {Treatment} from './treatment' export {TrialSchema, TrialResultSchema, run as runTrial, compare as compareTrial} from './trial' export type {Trial, TrialResult} from './trial' + +export { + BenchmarkPlanSchema, + ExperimentPlanSchema, + PLAN_VERSION, + PlanSchema, + create as createPlan, + deserialize as deserializePlan, + isBenchmarkPlan, + select as selectPlan, + serialize as serializePlan, +} from './plan' +export type { + BenchmarkPlan, + BenchmarkPlanTrialReference, + CreatePlanInput, + ExperimentPlan, + ExperimentPlanTrialReference, + Plan, + PlanTrialReference, + RuntimePlan, +} from './plan' diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts new file mode 100644 index 00000000..a9e4a9be --- /dev/null +++ b/packages/agent-eval/src/output.test.ts @@ -0,0 +1,160 @@ +import {expect, test} from 'vitest' +import {output as getBenchmarkOutput, write as writeBenchmarkOutput, type BenchmarkOutput} from './benchmark' +import {output as getExperimentOutput, write as writeExperimentOutput, type ExperimentOutput} from './experiment' +import {VirtualHost} from './host' +import {mergeShardOutputs} from './output' +import type {TrialResult} from './trial' + +async function writeBenchmarkShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeBenchmarkOutput(filepath, output, {host}) + return filepath + }), + ) +} + +async function writeExperimentShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeExperimentOutput(filepath, output, {host}) + return filepath + }), + ) +} + +test('detects and merges benchmark shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('benchmark', []), + getBenchmarkOutput('benchmark', []), + ]) + const merged = await mergeShardOutputs(filepaths, {host}) + + expect(merged.kind).toBe('benchmark') + expect(merged.output).toEqual({ + benchmarkId: 'benchmark', + capabilities: new Map(), + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + }) +}) + +test('detects and merges experiment shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [ + getExperimentOutput('experiment', []), + getExperimentOutput('experiment', []), + ]) + const merged = await mergeShardOutputs(filepaths, {host}) + + expect(merged.kind).toBe('experiment') + expect(merged.output).toEqual({ + experimentId: 'experiment', + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + }) +}) + +test('rejects mixed output types', async () => { + const host = VirtualHost.create() + await writeBenchmarkOutput('/bundle/output-1.json', getBenchmarkOutput('benchmark', []), {host}) + await writeExperimentOutput('/bundle/output-2.json', getExperimentOutput('experiment', []), {host}) + + await expect( + mergeShardOutputs(['/bundle/output-1.json', '/bundle/output-2.json'], { + host, + }), + ).rejects.toThrow('Cannot merge benchmark and experiment shard outputs together') +}) + +test('requires shard outputs from one source id', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('first', []), + getBenchmarkOutput('second', []), + ]) + + await expect(mergeShardOutputs(filepaths, {host})).rejects.toThrow( + 'Cannot merge benchmark outputs for different sources', + ) +}) + +test('rebases portable artifact paths when the merged output uses a different directory', async () => { + const trialResult: TrialResult = { + artifacts: { + directory: '/bundle/shards/artifacts/trial', + copilotConfigDirectory: '/bundle/shards/artifacts/trial/.copilot', + skillsConfigDirectory: '/bundle/shards/artifacts/trial/.agents', + testResultsPath: '/bundle/shards/artifacts/trial/workspace/test-results.json', + workspaceDirectory: '/bundle/shards/artifacts/trial/workspace', + }, + trial: { + id: 'trial', + scenario: { + id: 'scenario', + directory: '/scenarios/scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/scenario/scenario.test.ts', + }, + treatment: { + name: 'Control', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + }, + agent: { + sessions: [], + }, + testResults: { + numTotalTests: 0, + numPassedTests: 0, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }, + walkthrough: { + type: 'Screenshot', + filepath: '/bundle/shards/artifacts/trial/walkthrough/screenshot.png', + }, + } + const host = VirtualHost.create() + const filepath = '/bundle/shards/output-1.json' + await writeExperimentOutput( + filepath, + getExperimentOutput('experiment', [trialResult], { + baseDirectory: '/bundle/shards', + }), + {host}, + ) + + const merged = await mergeShardOutputs([filepath], { + host, + targetDirectory: '/bundle', + }) + + if (merged.kind !== 'experiment') { + throw new Error('Expected experiment output') + } + + expect(merged.output.trials.get('trial')).toEqual( + expect.objectContaining({ + artifacts: expect.objectContaining({ + directory: 'shards/artifacts/trial', + }), + walkthrough: { + type: 'Screenshot', + filepath: 'shards/artifacts/trial/walkthrough/screenshot.png', + }, + }), + ) +}) diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts new file mode 100644 index 00000000..952d6eb6 --- /dev/null +++ b/packages/agent-eval/src/output.ts @@ -0,0 +1,174 @@ +import path from 'node:path' +import {merge as mergeBenchmarkOutputs, read as readBenchmarkOutput, type BenchmarkOutput} from './benchmark' +import {merge as mergeExperimentOutputs, read as readExperimentOutput, type ExperimentOutput} from './experiment' +import {DefaultHost, type Host} from './host' +import type {TrialResult} from './trial' + +type MergedOutput = + | { + kind: 'benchmark' + output: BenchmarkOutput + } + | { + kind: 'experiment' + output: ExperimentOutput + } + +type MergeShardOutputOptions = { + host?: Host + targetDirectory?: string +} + +async function mergeShardOutputs( + filepaths: Array, + options: MergeShardOutputOptions = {}, +): Promise { + if (filepaths.length === 0) { + throw new Error('No shard outputs were found to merge') + } + + const host = options.host ?? DefaultHost + const manifests = await Promise.all( + filepaths.map(async filepath => { + return JSON.parse(await host.fs.readFile(filepath, 'utf-8')) as unknown + }), + ) + const kinds = manifests.map(getOutputKind) + const firstKind = kinds[0] + if ( + kinds.some(kind => { + return kind !== firstKind + }) + ) { + throw new Error('Cannot merge benchmark and experiment shard outputs together') + } + if (firstKind === 'benchmark') { + const outputs = await Promise.all( + filepaths.map(async filepath => { + const output = await readBenchmarkOutput(filepath, {host}) + return rebaseBenchmarkOutput(output, path.dirname(filepath), options.targetDirectory) + }), + ) + const output = mergeBenchmarkOutputs(outputs) + return { + kind: 'benchmark', + output, + } + } + + const outputs = await Promise.all( + filepaths.map(async filepath => { + const output = await readExperimentOutput(filepath, {host}) + return rebaseExperimentOutput(output, path.dirname(filepath), options.targetDirectory) + }), + ) + const output = mergeExperimentOutputs(outputs) + return { + kind: 'experiment', + output, + } +} + +function getOutputKind(input: unknown): 'benchmark' | 'experiment' { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + if (typeof parsed !== 'object' || parsed === null) { + throw new Error('Shard output must be a JSON object') + } + + const hasBenchmarkId = 'benchmarkId' in parsed + const hasExperimentId = 'experimentId' in parsed + if (hasBenchmarkId === hasExperimentId) { + throw new Error('Shard output must contain exactly one of benchmarkId or experimentId') + } + + return hasBenchmarkId ? 'benchmark' : 'experiment' +} + +function rebaseBenchmarkOutput( + output: BenchmarkOutput, + sourceDirectory: string, + targetDirectory?: string, +): BenchmarkOutput { + if (!sourceDirectory || !targetDirectory) { + return output + } + + for (const [id, trial] of output.trials) { + output.trials.set(id, { + ...trial, + artifacts: rebaseArtifacts(trial.artifacts, sourceDirectory, targetDirectory), + walkthrough: rebaseWalkthrough(trial.walkthrough, sourceDirectory, targetDirectory), + }) + } + + return output +} + +function rebaseExperimentOutput( + output: ExperimentOutput, + sourceDirectory: string, + targetDirectory?: string, +): ExperimentOutput { + if (!sourceDirectory || !targetDirectory) { + return output + } + + for (const [id, trial] of output.trials) { + output.trials.set(id, { + ...trial, + artifacts: rebaseArtifacts(trial.artifacts, sourceDirectory, targetDirectory), + walkthrough: rebaseWalkthrough(trial.walkthrough, sourceDirectory, targetDirectory), + }) + } + + return output +} + +function rebaseArtifacts( + artifacts: TrialResult['artifacts'], + sourceDirectory: string, + targetDirectory: string, +): TrialResult['artifacts'] { + return { + directory: rebasePath(artifacts.directory, sourceDirectory, targetDirectory), + copilotConfigDirectory: rebasePath(artifacts.copilotConfigDirectory, sourceDirectory, targetDirectory), + skillsConfigDirectory: rebasePath(artifacts.skillsConfigDirectory, sourceDirectory, targetDirectory), + testResultsPath: rebasePath(artifacts.testResultsPath, sourceDirectory, targetDirectory), + workspaceDirectory: rebasePath(artifacts.workspaceDirectory, sourceDirectory, targetDirectory), + } +} + +function rebaseWalkthrough( + walkthrough: TrialResult['walkthrough'], + sourceDirectory: string, + targetDirectory: string, +): TrialResult['walkthrough'] { + if (walkthrough.type === 'Screenshots') { + return { + ...walkthrough, + screenshots: walkthrough.screenshots.map(filepath => { + return rebasePath(filepath, sourceDirectory, targetDirectory) + }), + } + } + + if (walkthrough.type === 'Screenshot' || walkthrough.type === 'Video') { + return { + ...walkthrough, + filepath: rebasePath(walkthrough.filepath, sourceDirectory, targetDirectory), + } + } + + return walkthrough +} + +function rebasePath(filepath: string, sourceDirectory: string, targetDirectory: string): string { + if (path.isAbsolute(filepath)) { + return filepath + } + + return path.relative(targetDirectory, path.resolve(sourceDirectory, filepath)).split(path.sep).join(path.posix.sep) +} + +export {mergeShardOutputs} +export type {MergedOutput, MergeShardOutputOptions} diff --git a/packages/agent-eval/src/plan.test.ts b/packages/agent-eval/src/plan.test.ts index f85de6f8..e5d14dab 100644 --- a/packages/agent-eval/src/plan.test.ts +++ b/packages/agent-eval/src/plan.test.ts @@ -1,6 +1,6 @@ import {afterEach, describe, expect, test, vi} from 'vitest' import {VirtualHost} from './host' -import {create, run} from './plan' +import {create, deserialize, isBenchmarkPlan, run, select, serialize} from './plan' import {run as runTrial} from './trial' import type {Trial, TrialResult} from './trial' @@ -65,16 +65,92 @@ function createResult(trial: Trial): TrialResult { } } +function createReference(id: string) { + return { + id, + scenarioId: 'scenario', + treatmentId: 'Control', + model: { + name: 'gpt-5.6-sol' as const, + reasoningEffort: 'medium' as const, + }, + } +} + describe('create', () => { test('randomizes trials without mutating the input', async () => { - const trials = [createTrial('one'), createTrial('two'), createTrial('three')] + const trials = [createReference('one'), createReference('two'), createReference('three')] vi.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(0) - const plan = await create(trials) + const plan = create({ + source: { + kind: 'experiment', + id: 'test', + }, + trials, + }) expect(plan.trials.map(trial => trial.id)).toEqual(['two', 'three', 'one']) expect(trials.map(trial => trial.id)).toEqual(['one', 'two', 'three']) }) + + test('serializes and deserializes the durable plan without changing order or ids', () => { + const plan = create({ + source: { + kind: 'experiment', + id: 'test', + }, + trials: [createReference('one'), createReference('two')], + }) + + const restored = deserialize(serialize(plan)) + + expect(restored).toEqual(plan) + expect(restored.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(JSON.parse(serialize(plan))).toEqual(plan) + expect(serialize(plan)).not.toContain('setup') + }) + + test('rejects invalid durable plans and duplicate trial ids', () => { + expect(() => { + deserialize({ + version: 1, + source: { + kind: 'benchmark', + id: 'test', + }, + trials: [createReference('trial')], + }) + }).toThrow() + + expect(() => { + deserialize({ + version: 1, + source: { + kind: 'experiment', + id: 'test', + }, + trials: [createReference('trial'), createReference('trial')], + }) + }).toThrow('Plan contains duplicate trial id: trial') + }) + + test('selects deterministic shards from durable plan order', () => { + const plan = deserialize({ + version: 1, + source: { + kind: 'experiment', + id: 'test', + }, + trials: ['one', 'two', 'three', 'four', 'five'].map(createReference), + }) + + if (isBenchmarkPlan(plan)) { + throw new Error('Expected experiment plan') + } + + expect(select(plan, {order: 2, total: 3}).trials.map(trial => trial.id)).toEqual(['two', 'five']) + }) }) describe('run', () => { diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts index 9d503cc8..fa91c7f4 100644 --- a/packages/agent-eval/src/plan.ts +++ b/packages/agent-eval/src/plan.ts @@ -1,28 +1,131 @@ import Queue from 'p-queue' -import {run as runTrial} from './trial' -import type {Trial, TrialResult} from './trial' +import * as z from 'zod/mini' import type {EnvironmentConfig} from './environment' import {DefaultHost, type Host} from './host' import {logger} from './logger' +import {ModelVariantSchema} from './model' +import {selectShard, type Shard} from './shard' +import {run as runTrial} from './trial' +import type {Trial, TrialResult} from './trial' + +const PLAN_VERSION = 1 + +const BenchmarkPlanTrialReferenceSchema = z.object({ + id: z.string(), + scenarioId: z.string(), + treatmentId: z.string(), + model: ModelVariantSchema, + capabilityId: z.string(), +}) + +const ExperimentPlanTrialReferenceSchema = z.object({ + id: z.string(), + scenarioId: z.string(), + treatmentId: z.string(), + model: ModelVariantSchema, +}) + +const BenchmarkPlanSchema = z.object({ + version: z.literal(PLAN_VERSION), + source: z.object({ + kind: z.literal('benchmark'), + id: z.string(), + }), + trials: z.array(BenchmarkPlanTrialReferenceSchema), +}) + +const ExperimentPlanSchema = z.object({ + version: z.literal(PLAN_VERSION), + source: z.object({ + kind: z.literal('experiment'), + id: z.string(), + }), + trials: z.array(ExperimentPlanTrialReferenceSchema), +}) -/** - * A plan is an ordered list of trials to be ran. - */ -type Plan = { +const PlanSchema = z.union([BenchmarkPlanSchema, ExperimentPlanSchema]) + +type BenchmarkPlanTrialReference = z.infer +type ExperimentPlanTrialReference = z.infer +type BenchmarkPlan = z.infer +type ExperimentPlan = z.infer + +type Plan = BenchmarkPlan | ExperimentPlan +type PlanTrialReference = BenchmarkPlanTrialReference | ExperimentPlanTrialReference +type CreatePlanInput = Omit | Omit + +type RuntimePlan = { trials: Array } -// TODO: support plan with sharding -async function create(trials: Array): Promise { +function create(input: Omit): BenchmarkPlan +function create(input: Omit): ExperimentPlan +function create(input: CreatePlanInput): Plan { + const plan = + input.source.kind === 'benchmark' + ? BenchmarkPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) + : ExperimentPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) + assertUniqueTrialIds(plan) + return plan +} + +function serialize(plan: Plan): string { + const parsed = PlanSchema.parse(plan) + assertUniqueTrialIds(parsed) + return `${JSON.stringify(parsed, null, 2)}\n` +} + +function deserialize(input: unknown): Plan { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + const plan = PlanSchema.parse(parsed, {reportInput: true}) + assertUniqueTrialIds(plan) + return plan +} + +function select(plan: BenchmarkPlan, shard: Shard): BenchmarkPlan +function select(plan: ExperimentPlan, shard: Shard): ExperimentPlan +function select(plan: Plan, shard: Shard): Plan { + if (isBenchmarkPlan(plan)) { + return { + ...plan, + trials: selectShard(plan.trials, shard), + } + } + return { - trials: randomize(trials), + ...plan, + trials: selectShard(plan.trials, shard), + } +} + +function isBenchmarkPlan(plan: Plan): plan is BenchmarkPlan { + return plan.source.kind === 'benchmark' +} + +function assertUniqueTrialIds(plan: Plan): void { + const ids = new Set() + + for (const trial of plan.trials) { + if (ids.has(trial.id)) { + throw new Error(`Plan contains duplicate trial id: ${trial.id}`) + } + + ids.add(trial.id) } } function randomize(input: Array): Array { const randomized: Array = input.slice() - // Fisher–Yates shuffle + // Fisher-Yates shuffle for (let i = randomized.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)) ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]] @@ -34,7 +137,7 @@ function randomize(input: Array): Array { type RunPlanOptions = { env: EnvironmentConfig host?: Host - plan: Plan + plan: RuntimePlan } async function run({env, host = DefaultHost, plan}: RunPlanOptions): Promise> { @@ -76,5 +179,25 @@ async function retry(fn: () => Promise, retries: number = 3): Promise { } } -export {create, run} -export type {Plan} +export { + BenchmarkPlanSchema, + ExperimentPlanSchema, + PLAN_VERSION, + PlanSchema, + create, + deserialize, + isBenchmarkPlan, + run, + select, + serialize, +} +export type { + BenchmarkPlan, + BenchmarkPlanTrialReference, + CreatePlanInput, + ExperimentPlan, + ExperimentPlanTrialReference, + Plan, + PlanTrialReference, + RuntimePlan, +} diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh index f0cae005..4d45f940 100755 --- a/script/run-benchmark.sh +++ b/script/run-benchmark.sh @@ -4,13 +4,15 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [run|plan|shard|merge]" >&2 exit 1 fi benchmark_name="$1" +mode="${2:-run}" run_date="${RUN_DATE:-$(date -u +%F)}" run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date" +plan_path="$run_directory/plan.json" if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then echo "Benchmark name must be a file name without its extension" >&2 @@ -22,10 +24,48 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then exit 1 fi -node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --benchmark "$benchmark_name" \ - --benchmarks "$repository_root/benchmarks" \ - --concurrency "${CONCURRENCY:-1}" \ - --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ - --output-dir "$run_directory" \ - --scenarios "$repository_root/scenarios" +case "$mode" in + run) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmark "$benchmark_name" \ + --benchmarks "$repository_root/benchmarks" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" + ;; + plan) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmark "$benchmark_name" \ + --benchmarks "$repository_root/benchmarks" \ + --plan "$plan_path" \ + --scenarios "$repository_root/scenarios" + ;; + shard) + if [[ ! "${SHARD:-}" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + echo "SHARD must use the order/total format" >&2 + exit 1 + fi + + shard_order="${BASH_REMATCH[1]}" + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --artifacts "$run_directory/artifacts" \ + --benchmarks "$repository_root/benchmarks" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --from-plan "$plan_path" \ + --output "$run_directory/output-$shard_order.json" \ + --scenarios "$repository_root/scenarios" \ + --shard "$SHARD" + ;; + merge) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --merge-shards "$run_directory" \ + --output "$run_directory/output.json" + rm -f "$run_directory"/output-*.json + ;; + *) + echo "Mode must be one of: run, plan, shard, merge" >&2 + exit 1 + ;; +esac diff --git a/script/run-experiment.sh b/script/run-experiment.sh index e0123f3c..5433938b 100755 --- a/script/run-experiment.sh +++ b/script/run-experiment.sh @@ -4,14 +4,16 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [run|plan|shard|merge]" >&2 exit 1 fi experiment_name="$1" +mode="${2:-run}" run_date="${RUN_DATE:-$(date -u +%F)}" run_directory="$repository_root/results/experiments/$experiment_name/$run_date" +plan_path="$run_directory/plan.json" if [[ ! "$experiment_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then echo "Experiment name must be a file name without its extension" >&2 @@ -23,10 +25,48 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then exit 1 fi -node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --experiment "$experiment_name" \ - --experiments "$repository_root/experiments" \ - --concurrency "${CONCURRENCY:-1}" \ - --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ - --output-dir "$run_directory" \ - --scenarios "$repository_root/scenarios" +case "$mode" in + run) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --experiment "$experiment_name" \ + --experiments "$repository_root/experiments" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" + ;; + plan) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --experiment "$experiment_name" \ + --experiments "$repository_root/experiments" \ + --plan "$plan_path" \ + --scenarios "$repository_root/scenarios" + ;; + shard) + if [[ ! "${SHARD:-}" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + echo "SHARD must use the order/total format" >&2 + exit 1 + fi + + shard_order="${BASH_REMATCH[1]}" + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --artifacts "$run_directory/artifacts" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --experiments "$repository_root/experiments" \ + --from-plan "$plan_path" \ + --output "$run_directory/output-$shard_order.json" \ + --scenarios "$repository_root/scenarios" \ + --shard "$SHARD" + ;; + merge) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --merge-shards "$run_directory" \ + --output "$run_directory/output.json" + rm -f "$run_directory"/output-*.json + ;; + *) + echo "Mode must be one of: run, plan, shard, merge" >&2 + exit 1 + ;; +esac From 5eb45072265038f3d44424698168a07e8e291f6b Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 22:42:03 -0500 Subject: [PATCH 2/7] refactor: merge shard output manifests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/agent-eval/README.md | 3 + packages/agent-eval/src/benchmark.ts | 9 ++ packages/agent-eval/src/cli.ts | 6 +- packages/agent-eval/src/experiment.ts | 9 ++ packages/agent-eval/src/output.test.ts | 114 ++++++++---------- packages/agent-eval/src/output.ts | 153 ++++++++++--------------- 6 files changed, 130 insertions(+), 164 deletions(-) diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 108adac9..a962e6c4 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -196,6 +196,9 @@ result: agent-eval --merge-shards run --output run/output.json ``` +The merged `output.json` must stay in the same directory as the shard manifests +so their per-trial file references remain portable. + `--plan` and `--from-plan` default to `plan.json` when their path is omitted. `--shard` is only valid with `--from-plan`. Shard merging does not require a Copilot token. diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts index 4e2681f2..0f6c8347 100644 --- a/packages/agent-eval/src/benchmark.ts +++ b/packages/agent-eval/src/benchmark.ts @@ -394,6 +394,8 @@ const BenchmarkOutputFileSchema = z.object({ trials: z.record(z.string(), z.string()), }) +type BenchmarkOutputFile = z.infer + type BenchmarkOutput = { benchmarkId: string capabilities: Map> @@ -497,6 +499,11 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } +function parseOutputFile(input: unknown): BenchmarkOutputFile { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + return BenchmarkOutputFileSchema.parse(parsed, {reportInput: true}) +} + function merge(outputs: Array): BenchmarkOutput { const [first, ...remaining] = outputs if (!first) { @@ -553,6 +560,7 @@ export { listBenchmarks, merge, output, + parseOutputFile, read, resolvePlan, run, @@ -562,6 +570,7 @@ export type { BenchmarkConfig, Benchmark, BenchmarkOutput, + BenchmarkOutputFile, BenchmarkOutputOptions, BenchmarkRunResult, BenchmarkTrialResult, diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index ac8d1ecf..a0b881f9 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -199,11 +199,7 @@ if (mode.kind === 'create-plan') { await ensureParentDirectory(env.outputPath) logger.info('Writing merged %s output to: %s', merged.kind, env.outputPath) - if (merged.kind === 'benchmark') { - await writeBenchmarkOutput(env.outputPath, merged.output) - } else { - await writeExperimentOutput(env.outputPath, merged.output) - } + await fs.writeFile(env.outputPath, JSON.stringify(merged.output), 'utf-8') } else if (mode.kind === 'benchmark') { requireCopilotToken(COPILOT_GITHUB_TOKEN) logger.info('Running benchmark: %s', mode.id) diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts index fb5bfb00..26b2a865 100644 --- a/packages/agent-eval/src/experiment.ts +++ b/packages/agent-eval/src/experiment.ts @@ -358,6 +358,8 @@ const ExperimentOutputFileSchema = z.object({ trials: z.record(z.string(), z.string()), }) +type ExperimentOutputFile = z.infer + function output( experimentId: string, trialResults: ExperimentRunResult, @@ -438,6 +440,11 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } +function parseOutputFile(input: unknown): ExperimentOutputFile { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + return ExperimentOutputFileSchema.parse(parsed, {reportInput: true}) +} + function merge(outputs: Array): ExperimentOutput { const [first, ...remaining] = outputs if (!first) { @@ -492,6 +499,7 @@ export { listExperiments, merge, output, + parseOutputFile, read, resolvePlan, run, @@ -501,6 +509,7 @@ export type { ExperimentConfig, Experiment, ExperimentOutput, + ExperimentOutputFile, ExperimentOutputOptions, ExperimentScenarioConfig, InlineScenarioConfig, diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts index a9e4a9be..99c50368 100644 --- a/packages/agent-eval/src/output.test.ts +++ b/packages/agent-eval/src/output.test.ts @@ -3,7 +3,6 @@ import {output as getBenchmarkOutput, write as writeBenchmarkOutput, type Benchm import {output as getExperimentOutput, write as writeExperimentOutput, type ExperimentOutput} from './experiment' import {VirtualHost} from './host' import {mergeShardOutputs} from './output' -import type {TrialResult} from './trial' async function writeBenchmarkShards(host: VirtualHost, outputs: Array): Promise> { return Promise.all( @@ -36,10 +35,10 @@ test('detects and merges benchmark shard outputs', async () => { expect(merged.kind).toBe('benchmark') expect(merged.output).toEqual({ benchmarkId: 'benchmark', - capabilities: new Map(), - scenarios: new Map(), - treatments: new Map(), - trials: new Map(), + capabilities: {}, + scenarios: {}, + treatments: {}, + trials: {}, }) }) @@ -54,9 +53,9 @@ test('detects and merges experiment shard outputs', async () => { expect(merged.kind).toBe('experiment') expect(merged.output).toEqual({ experimentId: 'experiment', - scenarios: new Map(), - treatments: new Map(), - trials: new Map(), + scenarios: {}, + treatments: {}, + trials: {}, }) }) @@ -84,77 +83,56 @@ test('requires shard outputs from one source id', async () => { ) }) -test('rebases portable artifact paths when the merged output uses a different directory', async () => { - const trialResult: TrialResult = { - artifacts: { - directory: '/bundle/shards/artifacts/trial', - copilotConfigDirectory: '/bundle/shards/artifacts/trial/.copilot', - skillsConfigDirectory: '/bundle/shards/artifacts/trial/.agents', - testResultsPath: '/bundle/shards/artifacts/trial/workspace/test-results.json', - workspaceDirectory: '/bundle/shards/artifacts/trial/workspace', - }, - trial: { - id: 'trial', - scenario: { - id: 'scenario', - directory: '/scenarios/scenario', - prompt: 'Complete the task', - tags: [], - testPath: '/scenarios/scenario/scenario.test.ts', - }, - treatment: { - name: 'Control', +test('combines trial file references without reading the trial files', async () => { + const host = VirtualHost.create() + await host.fs.mkdir('/bundle', {recursive: true}) + await host.fs.writeFile( + '/bundle/output-1.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + first: 'artifacts/first/first.json', }, - model: { - name: 'gpt-5.6-sol', - reasoningEffort: 'medium', + }), + 'utf-8', + ) + await host.fs.writeFile( + '/bundle/output-2.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + second: 'artifacts/second/second.json', }, - }, - agent: { - sessions: [], - }, - testResults: { - numTotalTests: 0, - numPassedTests: 0, - numFailedTests: 0, - numPendingTests: 0, - numTodoTests: 0, - success: true, - testResults: [], - }, - walkthrough: { - type: 'Screenshot', - filepath: '/bundle/shards/artifacts/trial/walkthrough/screenshot.png', - }, - } - const host = VirtualHost.create() - const filepath = '/bundle/shards/output-1.json' - await writeExperimentOutput( - filepath, - getExperimentOutput('experiment', [trialResult], { - baseDirectory: '/bundle/shards', }), - {host}, + 'utf-8', ) - const merged = await mergeShardOutputs([filepath], { + const merged = await mergeShardOutputs(['/bundle/output-1.json', '/bundle/output-2.json'], { host, - targetDirectory: '/bundle', }) if (merged.kind !== 'experiment') { throw new Error('Expected experiment output') } - expect(merged.output.trials.get('trial')).toEqual( - expect.objectContaining({ - artifacts: expect.objectContaining({ - directory: 'shards/artifacts/trial', - }), - walkthrough: { - type: 'Screenshot', - filepath: 'shards/artifacts/trial/walkthrough/screenshot.png', - }, + expect(merged.output.trials).toEqual({ + first: 'artifacts/first/first.json', + second: 'artifacts/second/second.json', + }) +}) + +test('requires the merged output to stay beside the shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [getExperimentOutput('experiment', [])]) + + await expect( + mergeShardOutputs(filepaths, { + host, + targetDirectory: '/merged', }), - ) + ).rejects.toThrow('Shard outputs and the merged output must use the same directory') }) diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts index 952d6eb6..c38b3a74 100644 --- a/packages/agent-eval/src/output.ts +++ b/packages/agent-eval/src/output.ts @@ -1,17 +1,16 @@ import path from 'node:path' -import {merge as mergeBenchmarkOutputs, read as readBenchmarkOutput, type BenchmarkOutput} from './benchmark' -import {merge as mergeExperimentOutputs, read as readExperimentOutput, type ExperimentOutput} from './experiment' +import {parseOutputFile as parseBenchmarkOutputFile, type BenchmarkOutputFile} from './benchmark' +import {parseOutputFile as parseExperimentOutputFile, type ExperimentOutputFile} from './experiment' import {DefaultHost, type Host} from './host' -import type {TrialResult} from './trial' type MergedOutput = | { kind: 'benchmark' - output: BenchmarkOutput + output: BenchmarkOutputFile } | { kind: 'experiment' - output: ExperimentOutput + output: ExperimentOutputFile } type MergeShardOutputOptions = { @@ -27,6 +26,13 @@ async function mergeShardOutputs( throw new Error('No shard outputs were found to merge') } + const targetDirectory = path.resolve(options.targetDirectory ?? path.dirname(filepaths[0])) + for (const filepath of filepaths) { + if (path.resolve(path.dirname(filepath)) !== targetDirectory) { + throw new Error('Shard outputs and the merged output must use the same directory') + } + } + const host = options.host ?? DefaultHost const manifests = await Promise.all( filepaths.map(async filepath => { @@ -42,41 +48,27 @@ async function mergeShardOutputs( ) { throw new Error('Cannot merge benchmark and experiment shard outputs together') } + if (firstKind === 'benchmark') { - const outputs = await Promise.all( - filepaths.map(async filepath => { - const output = await readBenchmarkOutput(filepath, {host}) - return rebaseBenchmarkOutput(output, path.dirname(filepath), options.targetDirectory) - }), - ) - const output = mergeBenchmarkOutputs(outputs) return { kind: 'benchmark', - output, + output: mergeBenchmarkOutputFiles(manifests.map(parseBenchmarkOutputFile)), } } - const outputs = await Promise.all( - filepaths.map(async filepath => { - const output = await readExperimentOutput(filepath, {host}) - return rebaseExperimentOutput(output, path.dirname(filepath), options.targetDirectory) - }), - ) - const output = mergeExperimentOutputs(outputs) return { kind: 'experiment', - output, + output: mergeExperimentOutputFiles(manifests.map(parseExperimentOutputFile)), } } function getOutputKind(input: unknown): 'benchmark' | 'experiment' { - const parsed = typeof input === 'string' ? JSON.parse(input) : input - if (typeof parsed !== 'object' || parsed === null) { + if (typeof input !== 'object' || input === null) { throw new Error('Shard output must be a JSON object') } - const hasBenchmarkId = 'benchmarkId' in parsed - const hasExperimentId = 'experimentId' in parsed + const hasBenchmarkId = 'benchmarkId' in input + const hasExperimentId = 'experimentId' in input if (hasBenchmarkId === hasExperimentId) { throw new Error('Shard output must contain exactly one of benchmarkId or experimentId') } @@ -84,90 +76,69 @@ function getOutputKind(input: unknown): 'benchmark' | 'experiment' { return hasBenchmarkId ? 'benchmark' : 'experiment' } -function rebaseBenchmarkOutput( - output: BenchmarkOutput, - sourceDirectory: string, - targetDirectory?: string, -): BenchmarkOutput { - if (!sourceDirectory || !targetDirectory) { - return output +function mergeBenchmarkOutputFiles(outputs: Array): BenchmarkOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one benchmark output is required to merge shards') } - for (const [id, trial] of output.trials) { - output.trials.set(id, { - ...trial, - artifacts: rebaseArtifacts(trial.artifacts, sourceDirectory, targetDirectory), - walkthrough: rebaseWalkthrough(trial.walkthrough, sourceDirectory, targetDirectory), - }) + const result = structuredClone(first) + for (const output of remaining) { + if (output.benchmarkId !== result.benchmarkId) { + throw new Error( + `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${output.benchmarkId}"`, + ) + } + + mergeMetadataRecord(result.capabilities, output.capabilities, 'capability') + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) } - return output + return result } -function rebaseExperimentOutput( - output: ExperimentOutput, - sourceDirectory: string, - targetDirectory?: string, -): ExperimentOutput { - if (!sourceDirectory || !targetDirectory) { - return output - } - - for (const [id, trial] of output.trials) { - output.trials.set(id, { - ...trial, - artifacts: rebaseArtifacts(trial.artifacts, sourceDirectory, targetDirectory), - walkthrough: rebaseWalkthrough(trial.walkthrough, sourceDirectory, targetDirectory), - }) +function mergeExperimentOutputFiles(outputs: Array): ExperimentOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one experiment output is required to merge shards') } - return output -} + const result = structuredClone(first) + for (const output of remaining) { + if (output.experimentId !== result.experimentId) { + throw new Error( + `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${output.experimentId}"`, + ) + } -function rebaseArtifacts( - artifacts: TrialResult['artifacts'], - sourceDirectory: string, - targetDirectory: string, -): TrialResult['artifacts'] { - return { - directory: rebasePath(artifacts.directory, sourceDirectory, targetDirectory), - copilotConfigDirectory: rebasePath(artifacts.copilotConfigDirectory, sourceDirectory, targetDirectory), - skillsConfigDirectory: rebasePath(artifacts.skillsConfigDirectory, sourceDirectory, targetDirectory), - testResultsPath: rebasePath(artifacts.testResultsPath, sourceDirectory, targetDirectory), - workspaceDirectory: rebasePath(artifacts.workspaceDirectory, sourceDirectory, targetDirectory), + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) } + + return result } -function rebaseWalkthrough( - walkthrough: TrialResult['walkthrough'], - sourceDirectory: string, - targetDirectory: string, -): TrialResult['walkthrough'] { - if (walkthrough.type === 'Screenshots') { - return { - ...walkthrough, - screenshots: walkthrough.screenshots.map(filepath => { - return rebasePath(filepath, sourceDirectory, targetDirectory) - }), +function mergeMetadataRecord(target: Record, source: Record, type: string): void { + for (const [id, value] of Object.entries(source)) { + if (id in target && JSON.stringify(target[id]) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) } - } - if (walkthrough.type === 'Screenshot' || walkthrough.type === 'Video') { - return { - ...walkthrough, - filepath: rebasePath(walkthrough.filepath, sourceDirectory, targetDirectory), - } + target[id] = value } - - return walkthrough } -function rebasePath(filepath: string, sourceDirectory: string, targetDirectory: string): string { - if (path.isAbsolute(filepath)) { - return filepath - } +function mergeTrialReferences(target: Record, source: Record): void { + for (const [trialId, reference] of Object.entries(source)) { + if (trialId in target) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } - return path.relative(targetDirectory, path.resolve(sourceDirectory, filepath)).split(path.sep).join(path.posix.sep) + target[trialId] = reference + } } export {mergeShardOutputs} From cdae1c2fa5faffb8d8018c97cd90c0b8827a7733 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 22:48:43 -0500 Subject: [PATCH 3/7] refactor: use output-derived shard artifacts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 7eb8999b-2277-4d41-915f-fb35cac43373 --- packages/agent-eval/README.md | 1 - script/run-benchmark.sh | 1 - script/run-experiment.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index a962e6c4..af65c4cc 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -185,7 +185,6 @@ for each shard: COPILOT_GITHUB_TOKEN=... agent-eval \ --from-plan plan.json \ --shard 1/4 \ - --artifacts run/artifacts \ --output run/output-1.json ``` diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh index 4d45f940..c847e4ef 100755 --- a/script/run-benchmark.sh +++ b/script/run-benchmark.sh @@ -49,7 +49,6 @@ case "$mode" in shard_order="${BASH_REMATCH[1]}" node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --artifacts "$run_directory/artifacts" \ --benchmarks "$repository_root/benchmarks" \ --concurrency "${CONCURRENCY:-1}" \ --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ diff --git a/script/run-experiment.sh b/script/run-experiment.sh index 5433938b..0c69f84d 100755 --- a/script/run-experiment.sh +++ b/script/run-experiment.sh @@ -50,7 +50,6 @@ case "$mode" in shard_order="${BASH_REMATCH[1]}" node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --artifacts "$run_directory/artifacts" \ --concurrency "${CONCURRENCY:-1}" \ --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ --experiments "$repository_root/experiments" \ From a69561fd3e1784c4e9c994a04053bf3515a56c27 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 23:00:07 -0500 Subject: [PATCH 4/7] refactor: simplify shard result handling Use output directories to derive shard manifest names, rename the merge CLI mode, and keep result merging with durable plan logic. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2a354dad-88d9-4648-85d5-0a48ff1f03cf --- .changeset/bright-plans-shard.md | 2 +- packages/agent-eval/README.md | 8 +- packages/agent-eval/src/cli-options.test.ts | 8 +- packages/agent-eval/src/cli-options.ts | 22 +-- packages/agent-eval/src/cli.test.ts | 3 +- packages/agent-eval/src/cli.ts | 25 +-- packages/agent-eval/src/environment.test.ts | 22 +++ packages/agent-eval/src/environment.ts | 4 +- packages/agent-eval/src/index.ts | 3 + packages/agent-eval/src/output.test.ts | 138 ---------------- packages/agent-eval/src/output.ts | 145 ---------------- packages/agent-eval/src/plan.test.ts | 152 +++++++++++++++-- packages/agent-eval/src/plan.ts | 173 +++++++++++++++++--- script/run-benchmark.sh | 7 +- script/run-experiment.sh | 7 +- 15 files changed, 356 insertions(+), 363 deletions(-) delete mode 100644 packages/agent-eval/src/output.test.ts delete mode 100644 packages/agent-eval/src/output.ts diff --git a/.changeset/bright-plans-shard.md b/.changeset/bright-plans-shard.md index 2cd53377..463ec723 100644 --- a/.changeset/bright-plans-shard.md +++ b/.changeset/bright-plans-shard.md @@ -2,4 +2,4 @@ '@primer/agent-eval': minor --- -Add durable plan creation, replay, deterministic sharding, and shard output merging to the CLI and public package API. +Add durable plan creation, replay, deterministic sharding, and result merging to the CLI and public package API. diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index af65c4cc..a160b54f 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -185,22 +185,22 @@ for each shard: COPILOT_GITHUB_TOKEN=... agent-eval \ --from-plan plan.json \ --shard 1/4 \ - --output run/output-1.json + --output-dir run ``` After all shards finish, merge the `output-*.json` files into one portable result: ```sh -agent-eval --merge-shards run --output run/output.json +agent-eval --merge-results --output-dir run ``` The merged `output.json` must stay in the same directory as the shard manifests so their per-trial file references remain portable. `--plan` and `--from-plan` default to `plan.json` when their path is omitted. -`--shard` is only valid with `--from-plan`. Shard merging does not require a -Copilot token. +With `--output-dir`, `--shard 1/4` writes `output-1.json`. `--shard` is only +valid with `--from-plan`. Shard merging does not require a Copilot token. ## Scenario config authoring diff --git a/packages/agent-eval/src/cli-options.test.ts b/packages/agent-eval/src/cli-options.test.ts index 54b93433..63d04f5c 100644 --- a/packages/agent-eval/src/cli-options.test.ts +++ b/packages/agent-eval/src/cli-options.test.ts @@ -13,7 +13,6 @@ describe('normalizeOptionalPathArguments', () => { '--shard', '2/3', ]) - expect(normalizeOptionalPathArguments(['--merge-shards'])).toEqual(['--merge-shards=']) }) test('preserves explicit optional paths', () => { @@ -68,6 +67,13 @@ describe('getCliMode', () => { }) }).toThrow('--shard is only valid with --from-plan') + expect(() => { + getCliMode({ + experiment: 'experiment', + 'merge-results': true, + }) + }).toThrow('--merge-results cannot be combined') + expect(() => { getCliMode({ plan: 'plan.json', diff --git a/packages/agent-eval/src/cli-options.ts b/packages/agent-eval/src/cli-options.ts index 2e280df7..3fbe8ad9 100644 --- a/packages/agent-eval/src/cli-options.ts +++ b/packages/agent-eval/src/cli-options.ts @@ -3,7 +3,6 @@ const DEFAULT_PLAN_PATH = 'plan.json' const optionalPathDefaults = new Map([ ['--plan', DEFAULT_PLAN_PATH], ['--from-plan', DEFAULT_PLAN_PATH], - ['--merge-shards', ''], ]) type CliModeOptions = { @@ -11,7 +10,7 @@ type CliModeOptions = { experiment?: string plan?: string 'from-plan'?: string - 'merge-shards'?: string + 'merge-results'?: boolean shard?: string } @@ -39,8 +38,7 @@ type CliMode = shard?: string } | { - kind: 'merge-shards' - directory?: string + kind: 'merge-results' } function normalizeOptionalPathArguments(args: Array): Array { @@ -73,13 +71,8 @@ function getCliMode(options: CliModeOptions): CliMode { } if (options['from-plan']) { - if ( - options.benchmark || - options.experiment || - options.plan !== undefined || - options['merge-shards'] !== undefined - ) { - throw new Error('--from-plan cannot be combined with --benchmark, --experiment, --plan, or --merge-shards') + if (options.benchmark || options.experiment || options.plan !== undefined || options['merge-results']) { + throw new Error('--from-plan cannot be combined with --benchmark, --experiment, --plan, or --merge-results') } return { @@ -89,9 +82,9 @@ function getCliMode(options: CliModeOptions): CliMode { } } - if (options['merge-shards'] !== undefined) { + if (options['merge-results']) { if (options.benchmark || options.experiment || options.plan !== undefined) { - throw new Error('--merge-shards cannot be combined with --benchmark, --experiment, or --plan') + throw new Error('--merge-results cannot be combined with --benchmark, --experiment, or --plan') } if (options.shard) { @@ -99,8 +92,7 @@ function getCliMode(options: CliModeOptions): CliMode { } return { - kind: 'merge-shards', - directory: options['merge-shards'] || undefined, + kind: 'merge-results', } } diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts index 2a2579a6..c17904e7 100644 --- a/packages/agent-eval/src/cli.test.ts +++ b/packages/agent-eval/src/cli.test.ts @@ -26,6 +26,7 @@ describe('cli', () => { expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]')) expect(log).toHaveBeenCalledWith(expect.stringContaining('--output-dir ')) expect(log).toHaveBeenCalledWith(expect.stringContaining('--from-plan [path]')) + expect(log).toHaveBeenCalledWith(expect.stringContaining('--merge-results')) }) test('requires a Copilot token when running', async () => { @@ -62,7 +63,7 @@ describe('cli', () => { }) test('does not require a Copilot token when merging shard outputs', async () => { - process.argv = ['node', 'agent-eval', '--merge-shards', '/missing-agent-eval-shards'] + process.argv = ['node', 'agent-eval', '--merge-results', '--output-dir', '/missing-agent-eval-shards'] delete process.env.COPILOT_GITHUB_TOKEN await expect(import('./cli')).rejects.toThrow('ENOENT') diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index a0b881f9..bc8dd703 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -20,10 +20,10 @@ import { write as writeExperimentOutput, } from './experiment' import {logger} from './logger' -import {mergeShardOutputs} from './output' import { deserialize as deserializePlan, isBenchmarkPlan, + mergeResults, select as selectPlan, serialize as serializePlan, type BenchmarkPlan, @@ -94,8 +94,8 @@ const {values} = parseArgs({ type: 'string', description: 'Run trials from a durable plan', }, - 'merge-shards': { - type: 'string', + 'merge-results': { + type: 'boolean', description: 'Merge output-*.json shard outputs from a directory', }, shard: { @@ -122,7 +122,7 @@ Options: --output-dir The directory containing output.json and its artifacts --plan [path] Create a durable plan without running it (default: plan.json) --from-plan [path] Run trials from a durable plan (default: plan.json) - --merge-shards [dir] Merge output-*.json files (default: output file directory) + --merge-results Merge output-*.json files in --output-dir --scenarios The directory containing scenario directories (default: ./scenarios) --shard Select a deterministic shard from --from-plan `) @@ -140,6 +140,7 @@ if (values['log-level']) { const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY const mode = getCliMode(values) +const shard = mode.kind === 'from-plan' && mode.shard ? parseShard(mode.shard) : undefined const env = getEnvironmentConfig({ benchmarksDirectory: values.benchmarks, @@ -150,6 +151,7 @@ const env = getEnvironmentConfig({ outputDirectory: values['output-dir'], outputPath: values.output, scenariosDirectory: values.scenarios, + shard, }) logger.debug('Environment configuration: %o', env) @@ -177,8 +179,8 @@ if (mode.kind === 'create-plan') { await ensureParentDirectory(planPath) logger.info('Writing plan to: %s', planPath) await fs.writeFile(planPath, serializePlan(plan), 'utf-8') -} else if (mode.kind === 'merge-shards') { - const directory = path.resolve(mode.directory ?? path.dirname(env.outputPath)) +} else if (mode.kind === 'merge-results') { + const directory = path.dirname(env.outputPath) const entries = await fs.readdir(directory, { withFileTypes: true, }) @@ -193,7 +195,7 @@ if (mode.kind === 'create-plan') { const inputs = filenames.map(filename => { return path.join(directory, filename) }) - const merged = await mergeShardOutputs(inputs, { + const merged = await mergeResults(inputs, { targetDirectory: path.dirname(env.outputPath), }) @@ -262,7 +264,7 @@ if (mode.kind === 'create-plan') { requireCopilotToken(COPILOT_GITHUB_TOKEN) const planPath = path.resolve(mode.path) const durablePlan = deserializePlan(await fs.readFile(planPath, 'utf-8')) - const plan = mode.shard ? selectDurablePlan(durablePlan, mode.shard) : durablePlan + const plan = shard ? selectDurablePlan(durablePlan, shard) : durablePlan if (isBenchmarkPlan(plan)) { await runBenchmarkFromPlan(plan) @@ -279,13 +281,12 @@ function requireCopilotToken(token: string | undefined): asserts token is string } } -function selectDurablePlan(plan: Plan, shardValue: string): Plan { - const shard = parseShard(shardValue) +function selectDurablePlan(plan: Plan, selectedShard: ReturnType): Plan { if (isBenchmarkPlan(plan)) { - return selectPlan(plan, shard) + return selectPlan(plan, selectedShard) } - return selectPlan(plan, shard) + return selectPlan(plan, selectedShard) } async function ensureParentDirectory(filepath: string): Promise { diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts index b6ded377..c9253c02 100644 --- a/packages/agent-eval/src/environment.test.ts +++ b/packages/agent-eval/src/environment.test.ts @@ -62,6 +62,28 @@ describe('getEnvironmentConfig', () => { }) }) + test('derives the output filename from the shard within an output directory', () => { + expect( + getEnvironmentConfig({ + copilotToken: 'token', + outputDirectory: './results/run', + shard: { + order: 2, + total: 4, + }, + }), + ).toEqual({ + artifactsDirectory: path.resolve('results/run/artifacts'), + benchmarksDirectory: path.resolve('benchmarks'), + concurrency: 1, + copilotToken: 'token', + dockerImage: DEFAULT_DOCKER_IMAGE, + experimentsDirectory: path.resolve('experiments'), + outputPath: path.resolve('results/run/output-2.json'), + scenariosDirectory: path.resolve('scenarios'), + }) + }) + test('rejects output directory combinations with explicit output paths', () => { expect(() => { getEnvironmentConfig({ diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts index 8ce00d19..f24868db 100644 --- a/packages/agent-eval/src/environment.ts +++ b/packages/agent-eval/src/environment.ts @@ -1,5 +1,6 @@ import path from 'node:path' import {DEFAULT_DOCKER_IMAGE} from './sandbox' +import type {Shard} from './shard' type EnvironmentConfig = { artifactsDirectory: string @@ -21,6 +22,7 @@ type EnvironmentOptions = { outputDirectory?: string outputPath?: string scenariosDirectory?: string + shard?: Shard } function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig { @@ -37,7 +39,7 @@ function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig { : 1 const experimentsDirectory = path.resolve(options.experimentsDirectory ?? 'experiments') const outputPath = outputDirectory - ? path.join(outputDirectory, 'output.json') + ? path.join(outputDirectory, options.shard ? `output-${options.shard.order}.json` : 'output.json') : path.resolve(options.outputPath ?? 'output.json') const artifactsDirectory = path.join(path.dirname(outputPath), 'artifacts') const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios') diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index 4ca40efa..8f39084e 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -53,6 +53,7 @@ export { create as createPlan, deserialize as deserializePlan, isBenchmarkPlan, + mergeResults as mergePlanResults, select as selectPlan, serialize as serializePlan, } from './plan' @@ -62,6 +63,8 @@ export type { CreatePlanInput, ExperimentPlan, ExperimentPlanTrialReference, + MergedResults, + MergeResultsOptions, Plan, PlanTrialReference, RuntimePlan, diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts deleted file mode 100644 index 99c50368..00000000 --- a/packages/agent-eval/src/output.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import {expect, test} from 'vitest' -import {output as getBenchmarkOutput, write as writeBenchmarkOutput, type BenchmarkOutput} from './benchmark' -import {output as getExperimentOutput, write as writeExperimentOutput, type ExperimentOutput} from './experiment' -import {VirtualHost} from './host' -import {mergeShardOutputs} from './output' - -async function writeBenchmarkShards(host: VirtualHost, outputs: Array): Promise> { - return Promise.all( - outputs.map(async (output, index) => { - const filepath = `/bundle/output-${index + 1}.json` - await writeBenchmarkOutput(filepath, output, {host}) - return filepath - }), - ) -} - -async function writeExperimentShards(host: VirtualHost, outputs: Array): Promise> { - return Promise.all( - outputs.map(async (output, index) => { - const filepath = `/bundle/output-${index + 1}.json` - await writeExperimentOutput(filepath, output, {host}) - return filepath - }), - ) -} - -test('detects and merges benchmark shard outputs', async () => { - const host = VirtualHost.create() - const filepaths = await writeBenchmarkShards(host, [ - getBenchmarkOutput('benchmark', []), - getBenchmarkOutput('benchmark', []), - ]) - const merged = await mergeShardOutputs(filepaths, {host}) - - expect(merged.kind).toBe('benchmark') - expect(merged.output).toEqual({ - benchmarkId: 'benchmark', - capabilities: {}, - scenarios: {}, - treatments: {}, - trials: {}, - }) -}) - -test('detects and merges experiment shard outputs', async () => { - const host = VirtualHost.create() - const filepaths = await writeExperimentShards(host, [ - getExperimentOutput('experiment', []), - getExperimentOutput('experiment', []), - ]) - const merged = await mergeShardOutputs(filepaths, {host}) - - expect(merged.kind).toBe('experiment') - expect(merged.output).toEqual({ - experimentId: 'experiment', - scenarios: {}, - treatments: {}, - trials: {}, - }) -}) - -test('rejects mixed output types', async () => { - const host = VirtualHost.create() - await writeBenchmarkOutput('/bundle/output-1.json', getBenchmarkOutput('benchmark', []), {host}) - await writeExperimentOutput('/bundle/output-2.json', getExperimentOutput('experiment', []), {host}) - - await expect( - mergeShardOutputs(['/bundle/output-1.json', '/bundle/output-2.json'], { - host, - }), - ).rejects.toThrow('Cannot merge benchmark and experiment shard outputs together') -}) - -test('requires shard outputs from one source id', async () => { - const host = VirtualHost.create() - const filepaths = await writeBenchmarkShards(host, [ - getBenchmarkOutput('first', []), - getBenchmarkOutput('second', []), - ]) - - await expect(mergeShardOutputs(filepaths, {host})).rejects.toThrow( - 'Cannot merge benchmark outputs for different sources', - ) -}) - -test('combines trial file references without reading the trial files', async () => { - const host = VirtualHost.create() - await host.fs.mkdir('/bundle', {recursive: true}) - await host.fs.writeFile( - '/bundle/output-1.json', - JSON.stringify({ - experimentId: 'experiment', - scenarios: {}, - treatments: {}, - trials: { - first: 'artifacts/first/first.json', - }, - }), - 'utf-8', - ) - await host.fs.writeFile( - '/bundle/output-2.json', - JSON.stringify({ - experimentId: 'experiment', - scenarios: {}, - treatments: {}, - trials: { - second: 'artifacts/second/second.json', - }, - }), - 'utf-8', - ) - - const merged = await mergeShardOutputs(['/bundle/output-1.json', '/bundle/output-2.json'], { - host, - }) - - if (merged.kind !== 'experiment') { - throw new Error('Expected experiment output') - } - - expect(merged.output.trials).toEqual({ - first: 'artifacts/first/first.json', - second: 'artifacts/second/second.json', - }) -}) - -test('requires the merged output to stay beside the shard outputs', async () => { - const host = VirtualHost.create() - const filepaths = await writeExperimentShards(host, [getExperimentOutput('experiment', [])]) - - await expect( - mergeShardOutputs(filepaths, { - host, - targetDirectory: '/merged', - }), - ).rejects.toThrow('Shard outputs and the merged output must use the same directory') -}) diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts deleted file mode 100644 index c38b3a74..00000000 --- a/packages/agent-eval/src/output.ts +++ /dev/null @@ -1,145 +0,0 @@ -import path from 'node:path' -import {parseOutputFile as parseBenchmarkOutputFile, type BenchmarkOutputFile} from './benchmark' -import {parseOutputFile as parseExperimentOutputFile, type ExperimentOutputFile} from './experiment' -import {DefaultHost, type Host} from './host' - -type MergedOutput = - | { - kind: 'benchmark' - output: BenchmarkOutputFile - } - | { - kind: 'experiment' - output: ExperimentOutputFile - } - -type MergeShardOutputOptions = { - host?: Host - targetDirectory?: string -} - -async function mergeShardOutputs( - filepaths: Array, - options: MergeShardOutputOptions = {}, -): Promise { - if (filepaths.length === 0) { - throw new Error('No shard outputs were found to merge') - } - - const targetDirectory = path.resolve(options.targetDirectory ?? path.dirname(filepaths[0])) - for (const filepath of filepaths) { - if (path.resolve(path.dirname(filepath)) !== targetDirectory) { - throw new Error('Shard outputs and the merged output must use the same directory') - } - } - - const host = options.host ?? DefaultHost - const manifests = await Promise.all( - filepaths.map(async filepath => { - return JSON.parse(await host.fs.readFile(filepath, 'utf-8')) as unknown - }), - ) - const kinds = manifests.map(getOutputKind) - const firstKind = kinds[0] - if ( - kinds.some(kind => { - return kind !== firstKind - }) - ) { - throw new Error('Cannot merge benchmark and experiment shard outputs together') - } - - if (firstKind === 'benchmark') { - return { - kind: 'benchmark', - output: mergeBenchmarkOutputFiles(manifests.map(parseBenchmarkOutputFile)), - } - } - - return { - kind: 'experiment', - output: mergeExperimentOutputFiles(manifests.map(parseExperimentOutputFile)), - } -} - -function getOutputKind(input: unknown): 'benchmark' | 'experiment' { - if (typeof input !== 'object' || input === null) { - throw new Error('Shard output must be a JSON object') - } - - const hasBenchmarkId = 'benchmarkId' in input - const hasExperimentId = 'experimentId' in input - if (hasBenchmarkId === hasExperimentId) { - throw new Error('Shard output must contain exactly one of benchmarkId or experimentId') - } - - return hasBenchmarkId ? 'benchmark' : 'experiment' -} - -function mergeBenchmarkOutputFiles(outputs: Array): BenchmarkOutputFile { - const [first, ...remaining] = outputs - if (!first) { - throw new Error('At least one benchmark output is required to merge shards') - } - - const result = structuredClone(first) - for (const output of remaining) { - if (output.benchmarkId !== result.benchmarkId) { - throw new Error( - `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${output.benchmarkId}"`, - ) - } - - mergeMetadataRecord(result.capabilities, output.capabilities, 'capability') - mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') - mergeMetadataRecord(result.treatments, output.treatments, 'treatment') - mergeTrialReferences(result.trials, output.trials) - } - - return result -} - -function mergeExperimentOutputFiles(outputs: Array): ExperimentOutputFile { - const [first, ...remaining] = outputs - if (!first) { - throw new Error('At least one experiment output is required to merge shards') - } - - const result = structuredClone(first) - for (const output of remaining) { - if (output.experimentId !== result.experimentId) { - throw new Error( - `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${output.experimentId}"`, - ) - } - - mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') - mergeMetadataRecord(result.treatments, output.treatments, 'treatment') - mergeTrialReferences(result.trials, output.trials) - } - - return result -} - -function mergeMetadataRecord(target: Record, source: Record, type: string): void { - for (const [id, value] of Object.entries(source)) { - if (id in target && JSON.stringify(target[id]) !== JSON.stringify(value)) { - throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) - } - - target[id] = value - } -} - -function mergeTrialReferences(target: Record, source: Record): void { - for (const [trialId, reference] of Object.entries(source)) { - if (trialId in target) { - throw new Error(`Cannot merge duplicate trial id: ${trialId}`) - } - - target[trialId] = reference - } -} - -export {mergeShardOutputs} -export type {MergedOutput, MergeShardOutputOptions} diff --git a/packages/agent-eval/src/plan.test.ts b/packages/agent-eval/src/plan.test.ts index e5d14dab..d7a8d07e 100644 --- a/packages/agent-eval/src/plan.test.ts +++ b/packages/agent-eval/src/plan.test.ts @@ -1,6 +1,8 @@ import {afterEach, describe, expect, test, vi} from 'vitest' +import {output as getBenchmarkOutput, write as writeBenchmarkOutput, type BenchmarkOutput} from './benchmark' +import {output as getExperimentOutput, write as writeExperimentOutput, type ExperimentOutput} from './experiment' import {VirtualHost} from './host' -import {create, deserialize, isBenchmarkPlan, run, select, serialize} from './plan' +import {create, deserialize, isBenchmarkPlan, mergeResults, run, select, serialize} from './plan' import {run as runTrial} from './trial' import type {Trial, TrialResult} from './trial' @@ -77,6 +79,26 @@ function createReference(id: string) { } } +async function writeBenchmarkShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeBenchmarkOutput(filepath, output, {host}) + return filepath + }), + ) +} + +async function writeExperimentShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeExperimentOutput(filepath, output, {host}) + return filepath + }), + ) +} + describe('create', () => { test('randomizes trials without mutating the input', async () => { const trials = [createReference('one'), createReference('two'), createReference('three')] @@ -111,7 +133,7 @@ describe('create', () => { expect(serialize(plan)).not.toContain('setup') }) - test('rejects invalid durable plans and duplicate trial ids', () => { + test('rejects invalid durable plans', () => { expect(() => { deserialize({ version: 1, @@ -122,17 +144,6 @@ describe('create', () => { trials: [createReference('trial')], }) }).toThrow() - - expect(() => { - deserialize({ - version: 1, - source: { - kind: 'experiment', - id: 'test', - }, - trials: [createReference('trial'), createReference('trial')], - }) - }).toThrow('Plan contains duplicate trial id: trial') }) test('selects deterministic shards from durable plan order', () => { @@ -242,3 +253,118 @@ describe('run', () => { expect(runTrial).toHaveBeenCalledTimes(4) }) }) + +describe('mergeResults', () => { + test('detects and merges benchmark shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('benchmark', []), + getBenchmarkOutput('benchmark', []), + ]) + const merged = await mergeResults(filepaths, {host}) + + expect(merged.kind).toBe('benchmark') + expect(merged.output).toEqual({ + benchmarkId: 'benchmark', + capabilities: {}, + scenarios: {}, + treatments: {}, + trials: {}, + }) + }) + + test('detects and merges experiment shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [ + getExperimentOutput('experiment', []), + getExperimentOutput('experiment', []), + ]) + const merged = await mergeResults(filepaths, {host}) + + expect(merged.kind).toBe('experiment') + expect(merged.output).toEqual({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: {}, + }) + }) + + test('rejects mixed output types', async () => { + const host = VirtualHost.create() + await writeBenchmarkOutput('/bundle/output-1.json', getBenchmarkOutput('benchmark', []), {host}) + await writeExperimentOutput('/bundle/output-2.json', getExperimentOutput('experiment', []), {host}) + + await expect( + mergeResults(['/bundle/output-1.json', '/bundle/output-2.json'], { + host, + }), + ).rejects.toThrow('Cannot merge benchmark and experiment shard outputs together') + }) + + test('requires shard outputs from one source id', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('first', []), + getBenchmarkOutput('second', []), + ]) + + await expect(mergeResults(filepaths, {host})).rejects.toThrow( + 'Cannot merge benchmark outputs for different sources', + ) + }) + + test('combines trial file references without reading the trial files', async () => { + const host = VirtualHost.create() + await host.fs.mkdir('/bundle', {recursive: true}) + await host.fs.writeFile( + '/bundle/output-1.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + first: 'artifacts/first/first.json', + }, + }), + 'utf-8', + ) + await host.fs.writeFile( + '/bundle/output-2.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + second: 'artifacts/second/second.json', + }, + }), + 'utf-8', + ) + + const merged = await mergeResults(['/bundle/output-1.json', '/bundle/output-2.json'], { + host, + }) + + if (merged.kind !== 'experiment') { + throw new Error('Expected experiment output') + } + + expect(merged.output.trials).toEqual({ + first: 'artifacts/first/first.json', + second: 'artifacts/second/second.json', + }) + }) + + test('requires the merged output to stay beside the shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [getExperimentOutput('experiment', [])]) + + await expect( + mergeResults(filepaths, { + host, + targetDirectory: '/merged', + }), + ).rejects.toThrow('Shard outputs and the merged output must use the same directory') + }) +}) diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts index fa91c7f4..9b75bef5 100644 --- a/packages/agent-eval/src/plan.ts +++ b/packages/agent-eval/src/plan.ts @@ -1,6 +1,9 @@ +import path from 'node:path' import Queue from 'p-queue' import * as z from 'zod/mini' +import type {BenchmarkOutputFile} from './benchmark' import type {EnvironmentConfig} from './environment' +import type {ExperimentOutputFile} from './experiment' import {DefaultHost, type Host} from './host' import {logger} from './logger' import {ModelVariantSchema} from './model' @@ -58,36 +61,45 @@ type RuntimePlan = { trials: Array } +type MergedResults = + | { + kind: 'benchmark' + output: BenchmarkOutputFile + } + | { + kind: 'experiment' + output: ExperimentOutputFile + } + +type MergeResultsOptions = { + host?: Host + targetDirectory?: string +} + function create(input: Omit): BenchmarkPlan function create(input: Omit): ExperimentPlan function create(input: CreatePlanInput): Plan { - const plan = - input.source.kind === 'benchmark' - ? BenchmarkPlanSchema.parse({ - version: PLAN_VERSION, - source: input.source, - trials: randomize(input.trials), - }) - : ExperimentPlanSchema.parse({ - version: PLAN_VERSION, - source: input.source, - trials: randomize(input.trials), - }) - assertUniqueTrialIds(plan) - return plan + return input.source.kind === 'benchmark' + ? BenchmarkPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) + : ExperimentPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) } function serialize(plan: Plan): string { const parsed = PlanSchema.parse(plan) - assertUniqueTrialIds(parsed) return `${JSON.stringify(parsed, null, 2)}\n` } function deserialize(input: unknown): Plan { const parsed = typeof input === 'string' ? JSON.parse(input) : input - const plan = PlanSchema.parse(parsed, {reportInput: true}) - assertUniqueTrialIds(plan) - return plan + return PlanSchema.parse(parsed, {reportInput: true}) } function select(plan: BenchmarkPlan, shard: Shard): BenchmarkPlan @@ -110,15 +122,125 @@ function isBenchmarkPlan(plan: Plan): plan is BenchmarkPlan { return plan.source.kind === 'benchmark' } -function assertUniqueTrialIds(plan: Plan): void { - const ids = new Set() +async function mergeResults(filepaths: Array, options: MergeResultsOptions = {}): Promise { + if (filepaths.length === 0) { + throw new Error('No shard outputs were found to merge') + } + + const targetDirectory = path.resolve(options.targetDirectory ?? path.dirname(filepaths[0])) + for (const filepath of filepaths) { + if (path.resolve(path.dirname(filepath)) !== targetDirectory) { + throw new Error('Shard outputs and the merged output must use the same directory') + } + } + + const host = options.host ?? DefaultHost + const manifests = await Promise.all( + filepaths.map(async filepath => { + return JSON.parse(await host.fs.readFile(filepath, 'utf-8')) as unknown + }), + ) + const kinds = manifests.map(getOutputKind) + const firstKind = kinds[0] + if ( + kinds.some(kind => { + return kind !== firstKind + }) + ) { + throw new Error('Cannot merge benchmark and experiment shard outputs together') + } + + if (firstKind === 'benchmark') { + const {parseOutputFile} = await import('./benchmark') + return { + kind: 'benchmark', + output: mergeBenchmarkOutputFiles(manifests.map(parseOutputFile)), + } + } + + const {parseOutputFile} = await import('./experiment') + return { + kind: 'experiment', + output: mergeExperimentOutputFiles(manifests.map(parseOutputFile)), + } +} + +function getOutputKind(input: unknown): 'benchmark' | 'experiment' { + if (typeof input !== 'object' || input === null) { + throw new Error('Shard output must be a JSON object') + } + + const hasBenchmarkId = 'benchmarkId' in input + const hasExperimentId = 'experimentId' in input + if (hasBenchmarkId === hasExperimentId) { + throw new Error('Shard output must contain exactly one of benchmarkId or experimentId') + } + + return hasBenchmarkId ? 'benchmark' : 'experiment' +} + +function mergeBenchmarkOutputFiles(outputs: Array): BenchmarkOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one benchmark output is required to merge shards') + } + + const result = structuredClone(first) + for (const output of remaining) { + if (output.benchmarkId !== result.benchmarkId) { + throw new Error( + `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${output.benchmarkId}"`, + ) + } + + mergeMetadataRecord(result.capabilities, output.capabilities, 'capability') + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) + } + + return result +} + +function mergeExperimentOutputFiles(outputs: Array): ExperimentOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one experiment output is required to merge shards') + } + + const result = structuredClone(first) + for (const output of remaining) { + if (output.experimentId !== result.experimentId) { + throw new Error( + `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${output.experimentId}"`, + ) + } + + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) + } + + return result +} + +function mergeMetadataRecord(target: Record, source: Record, type: string): void { + for (const [id, value] of Object.entries(source)) { + if (id in target && JSON.stringify(target[id]) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target[id] = value + } +} - for (const trial of plan.trials) { - if (ids.has(trial.id)) { - throw new Error(`Plan contains duplicate trial id: ${trial.id}`) +function mergeTrialReferences(target: Record, source: Record): void { + for (const [trialId, reference] of Object.entries(source)) { + if (trialId in target) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) } - ids.add(trial.id) + target[trialId] = reference } } @@ -187,6 +309,7 @@ export { create, deserialize, isBenchmarkPlan, + mergeResults, run, select, serialize, @@ -197,6 +320,8 @@ export type { CreatePlanInput, ExperimentPlan, ExperimentPlanTrialReference, + MergedResults, + MergeResultsOptions, Plan, PlanTrialReference, RuntimePlan, diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh index c847e4ef..a3cb6c67 100755 --- a/script/run-benchmark.sh +++ b/script/run-benchmark.sh @@ -47,20 +47,19 @@ case "$mode" in exit 1 fi - shard_order="${BASH_REMATCH[1]}" node "$repository_root/packages/agent-eval/bin/agent-eval" \ --benchmarks "$repository_root/benchmarks" \ --concurrency "${CONCURRENCY:-1}" \ --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ --from-plan "$plan_path" \ - --output "$run_directory/output-$shard_order.json" \ + --output-dir "$run_directory" \ --scenarios "$repository_root/scenarios" \ --shard "$SHARD" ;; merge) node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --merge-shards "$run_directory" \ - --output "$run_directory/output.json" + --merge-results \ + --output-dir "$run_directory" rm -f "$run_directory"/output-*.json ;; *) diff --git a/script/run-experiment.sh b/script/run-experiment.sh index 0c69f84d..1e138cb0 100755 --- a/script/run-experiment.sh +++ b/script/run-experiment.sh @@ -48,20 +48,19 @@ case "$mode" in exit 1 fi - shard_order="${BASH_REMATCH[1]}" node "$repository_root/packages/agent-eval/bin/agent-eval" \ --concurrency "${CONCURRENCY:-1}" \ --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ --experiments "$repository_root/experiments" \ --from-plan "$plan_path" \ - --output "$run_directory/output-$shard_order.json" \ + --output-dir "$run_directory" \ --scenarios "$repository_root/scenarios" \ --shard "$SHARD" ;; merge) node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --merge-shards "$run_directory" \ - --output "$run_directory/output.json" + --merge-results \ + --output-dir "$run_directory" rm -f "$run_directory"/output-*.json ;; *) From 33f8de4872162f5b48038389b1b34f487fc22267 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 23:14:11 -0500 Subject: [PATCH 5/7] fix: preserve benchmark setup in durable plans Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/agent-eval/src/benchmark.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts index 0f6c8347..901d85c4 100644 --- a/packages/agent-eval/src/benchmark.ts +++ b/packages/agent-eval/src/benchmark.ts @@ -339,13 +339,13 @@ async function run({ } function createBenchmarkTreatment(benchmark: Benchmark, capability: Capability): Treatment { - const setup = - benchmark.setup || capability.setup - ? async ({sandbox}: Parameters[0]) => { - await benchmark.setup?.({sandbox}) - await capability.setup?.({sandbox}) - } - : undefined + let setup = benchmark.setup ?? capability.setup + if (benchmark.setup && capability.setup) { + setup = async ({sandbox}: Parameters[0]) => { + await benchmark.setup?.({sandbox}) + await capability.setup?.({sandbox}) + } + } return { name: 'Benchmark', From aa2b4a3fc9c2678c028f37bcd57de3b17c4fb942 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 23:18:26 -0500 Subject: [PATCH 6/7] fix: stabilize plan sharding checks Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/agent-eval/src/benchmark.test.ts | 3 +-- packages/agent-eval/src/cli.test.ts | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts index 67506b3d..dd6f5a5c 100644 --- a/packages/agent-eval/src/benchmark.test.ts +++ b/packages/agent-eval/src/benchmark.test.ts @@ -13,9 +13,8 @@ import { type Benchmark, type BenchmarkTrialResult, } from './benchmark' -import {deserialize as deserializePlan, isBenchmarkPlan} from './plan' import {VirtualHost} from './host' -import {run as runPlan} from './plan' +import {deserialize as deserializePlan, isBenchmarkPlan, run as runPlan} from './plan' import {defineConfig as defineScenarioConfig} from './scenario' vi.mock('./plan', async importOriginal => { diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts index c17904e7..d919fddd 100644 --- a/packages/agent-eval/src/cli.test.ts +++ b/packages/agent-eval/src/cli.test.ts @@ -1,3 +1,6 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' import {afterEach, describe, expect, test, vi} from 'vitest' const originalArgv = process.argv @@ -56,10 +59,16 @@ describe('cli', () => { }) test('does not require a Copilot token when creating a plan', async () => { - process.argv = ['node', 'agent-eval', '--benchmark', 'test', '--plan'] - delete process.env.COPILOT_GITHUB_TOKEN + const benchmarksDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-benchmarks-')) + + try { + process.argv = ['node', 'agent-eval', '--benchmark', 'test', '--benchmarks', benchmarksDirectory, '--plan'] + delete process.env.COPILOT_GITHUB_TOKEN - await expect(import('./cli')).rejects.toThrow('Benchmark "test" was not found') + await expect(import('./cli')).rejects.toThrow('Benchmark "test" was not found') + } finally { + await fs.rm(benchmarksDirectory, {recursive: true}) + } }) test('does not require a Copilot token when merging shard outputs', async () => { From 4c9bb02519cff9d3bdaa735a5eab58a331757733 Mon Sep 17 00:00:00 2001 From: Josh Black Date: Thu, 3 Sep 2026 23:38:27 -0500 Subject: [PATCH 7/7] fix: create sharded output directories Keep parent directory creation for plan and merged outputs without redundant existence checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 983b31cf-cd77-4cfe-854f-3c7fb34543a1 --- packages/agent-eval/src/cli.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index bc8dd703..82e27128 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -290,10 +290,7 @@ function selectDurablePlan(plan: Plan, selectedShard: ReturnType { - const directory = path.dirname(filepath) - if (!existsSync(directory)) { - await fs.mkdir(directory, {recursive: true}) - } + await fs.mkdir(path.dirname(filepath), {recursive: true}) } async function runBenchmarkFromPlan(plan: BenchmarkPlan): Promise {