diff --git a/.changeset/bright-benchmarks-run.md b/.changeset/bright-benchmarks-run.md new file mode 100644 index 00000000..0cee7d26 --- /dev/null +++ b/.changeset/bright-benchmarks-run.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add benchmark configuration, discovery, execution, and output APIs through `@primer/agent-eval/benchmark` and explicitly named package-root exports. The CLI can now select and run benchmarks from a benchmarks directory. diff --git a/.changeset/calm-experiments-unify.md b/.changeset/calm-experiments-unify.md new file mode 100644 index 00000000..647e5dca --- /dev/null +++ b/.changeset/calm-experiments-unify.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Move experiment APIs to `@primer/agent-eval/experiment`, replacing `findExperiment`, package-root `run`, and package-root `defineConfig` with `getExperiment`, `runExperiment`, and `defineExperimentConfig`. Remove the legacy experiment loading helpers in favor of `getExperiment` and `listExperiments` with explicit source directories. diff --git a/.changeset/fresh-experiment-configs.md b/.changeset/fresh-experiment-configs.md new file mode 100644 index 00000000..518205c9 --- /dev/null +++ b/.changeset/fresh-experiment-configs.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add `ExperimentConfigSchema` and update experiment configuration to use model variants, the new treatment API, and the new sandbox interface. Scenarios continue to support IDs and `{path, name?}` entries through the exported `ExperimentScenarioConfig` and `InlineScenarioConfig` types. diff --git a/.changeset/gentle-scenarios-unify.md b/.changeset/gentle-scenarios-unify.md new file mode 100644 index 00000000..ac210883 --- /dev/null +++ b/.changeset/gentle-scenarios-unify.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Move scenario APIs to `@primer/agent-eval/scenario`, replacing `defineScenario`, `findScenario`, and `ResolvedScenario` with `defineConfig`, `getScenario`, and `Scenario`. Add scenario schemas and remove the legacy `@primer/agent-eval/scenarios` entry point and `loadScenarioDirectory`. diff --git a/.changeset/happy-output-maps.md b/.changeset/happy-output-maps.md new file mode 100644 index 00000000..7b37fdef --- /dev/null +++ b/.changeset/happy-output-maps.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Replace `createAgentEvalOutput`, `parseAgentEvalOutput`, `AgentEvalOutput`, and `AgentEvalOutputResult` with the experiment `output` and `deserialize` helpers, `ExperimentOutput`, and `TrialResult`. Experiment output now uses keyed maps, model variants, per-session agent metrics, and directory-oriented artifact fields. diff --git a/.changeset/kind-trials-compare.md b/.changeset/kind-trials-compare.md new file mode 100644 index 00000000..a973361a --- /dev/null +++ b/.changeset/kind-trials-compare.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add package-root trial and treatment schemas, types, execution helpers, and comparison helpers, replacing `TreatmentResult` with `TrialResult`. Trials continue to run optional Playwright browser tests and combine them with standard scenario test results. diff --git a/.changeset/lucky-sandboxes-open.md b/.changeset/lucky-sandboxes-open.md new file mode 100644 index 00000000..07755484 --- /dev/null +++ b/.changeset/lucky-sandboxes-open.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add `@primer/agent-eval/sandbox` for sandbox runtimes, configuration, constants, and the plugin and MCP types previously exported from the experiment entry point. Replace the concrete `Sandbox` class with the `Sandbox` interface plus `SystemSandbox` and `VirtualSandbox`. diff --git a/.changeset/neat-models-vary.md b/.changeset/neat-models-vary.md new file mode 100644 index 00000000..58d3fe49 --- /dev/null +++ b/.changeset/neat-models-vary.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Replace `ExperimentModelConfig`, `ModelInfo`, and `resolveModelConfigs` with model variant configuration, schemas, and expansion helpers. Low-level model helpers and types are no longer exported from the package root or experiment entry point. diff --git a/.changeset/plain-scenarios-discover.md b/.changeset/plain-scenarios-discover.md new file mode 100644 index 00000000..bc3ec7e5 --- /dev/null +++ b/.changeset/plain-scenarios-discover.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Update scenario discovery to support explicit hosts, skip template directories, and recognize `browser.test.ts`. Inline scenario paths remain supported without package discovery requirements and continue to recognize `scenario.browser.test.ts`. diff --git a/.changeset/portable-results-bundle.md b/.changeset/portable-results-bundle.md new file mode 100644 index 00000000..074ebd31 --- /dev/null +++ b/.changeset/portable-results-bundle.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add `--output-dir` for portable experiment and benchmark bundles with artifact and walkthrough paths relative to `output.json`. Remove `--artifacts` and derive the artifact directory from `--output-dir` or the directory containing `--output`. diff --git a/.changeset/quick-copilot-metrics.md b/.changeset/quick-copilot-metrics.md new file mode 100644 index 00000000..5b340c1a --- /dev/null +++ b/.changeset/quick-copilot-metrics.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': patch +--- + +Accept sub-agent `user.message` events without `agentMode` and preserve their routing fields. Collect output token counts from `model.message` events while retaining compatibility with older Copilot output. diff --git a/.changeset/steady-trial-results.md b/.changeset/steady-trial-results.md new file mode 100644 index 00000000..dfc532b5 --- /dev/null +++ b/.changeset/steady-trial-results.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': patch +--- + +Exclude the temporary `agent-browser` walkthrough skill from downloaded trial artifacts. diff --git a/.changeset/tender-benchmarks-compare.md b/.changeset/tender-benchmarks-compare.md new file mode 100644 index 00000000..b46d1535 --- /dev/null +++ b/.changeset/tender-benchmarks-compare.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': patch +--- + +Compare benchmark test success rates instead of passed-test totals and report equal metrics as a 0% change. diff --git a/.changeset/tidy-cli-entrypoint.md b/.changeset/tidy-cli-entrypoint.md new file mode 100644 index 00000000..89038be2 --- /dev/null +++ b/.changeset/tidy-cli-entrypoint.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Remove the `@primer/agent-eval/cli` package entry point. Use the `agent-eval` executable for CLI usage and the package root or API entry points for programmatic usage. diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml deleted file mode 100644 index df7b7cc7..00000000 --- a/.github/workflows/baseline.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: baseline -on: - schedule: - - cron: '0 0 * * 1' - workflow_dispatch: - -concurrency: - group: weekly-baseline - cancel-in-progress: false - -permissions: - contents: read - -jobs: - setup: - runs-on: ubuntu-latest - outputs: - run-date: ${{ steps.run.outputs.date }} - steps: - - name: set run date - id: run - run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" - - run: - needs: setup - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - order: [1, 2, 3, 4] - steps: - - name: checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: set up pnpm - uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 - with: - cache: true - require-lockfile: true - - name: set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version-file: '.nvmrc' - - name: build project - run: pnpm run build - - name: run baseline - env: - CONCURRENCY: 2 - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - OUTPUT_FILE: output-${{ matrix.order }}.json - RUN_DATE: ${{ needs.setup.outputs.run-date }} - SHARD: ${{ matrix.order }}/${{ strategy.job-total }} - run: script/run-baseline.sh - - name: upload baseline results - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: baseline-shard-${{ github.run_id }}-${{ matrix.order }} - path: | - results - artifacts - if-no-files-found: error - retention-days: 1 - compression-level: 9 - - merge: - needs: run - runs-on: ubuntu-latest - steps: - - name: download baseline results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: baseline-shard-${{ github.run_id }}-* - merge-multiple: true - - name: merge baseline results - run: | - output_directory="$(dirname "$(find results -type f -name 'output-*.json' -print -quit)")" - jq --slurp ' - . as $outputs - | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments - | $outputs[0] - | .treatments = $treatments - | .results = [ - $outputs[] as $output - | $output.results[] - | . as $result - | ($output.treatments[] | select(.id == $result.treatmentId).config.name) as $name - | .treatmentId = ($treatments[] | select(.config.name == $name).id) - ] - ' "$output_directory"/output-*.json > "$output_directory/output.json" - rm "$output_directory"/output-*.json - - name: upload baseline results - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: baseline-${{ github.run_id }} - path: | - results - artifacts - if-no-files-found: error - retention-days: 90 - compression-level: 9 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..f762a1ab --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,65 @@ +name: benchmark +on: + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + inputs: + concurrency: + description: Number of trials to run in parallel + required: false + default: '1' + type: string + docker-image: + description: >- + Docker container image to use for running trials. + Must be a Debian-based Node image with apt-get and a node user (e.g. node:26.5.0-slim). + required: false + default: 'node:26.5.0-slim' + type: string + +concurrency: + group: weekly-design-system-benchmark + cancel-in-progress: false + +permissions: + contents: read + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: run benchmark + env: + CONCURRENCY: ${{ inputs.concurrency }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} + run: script/run-benchmark.sh + - name: prepare benchmark artifact + if: ${{ always() }} + run: | + mkdir -p workflow-artifact + if [[ -d results ]]; then + mv results workflow-artifact/results + fi + - name: upload benchmark results + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-${{ github.run_id }} + path: workflow-artifact + if-no-files-found: error + retention-days: 90 + compression-level: 9 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4fb02fc3..bd76352d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,7 +5,7 @@ on: - main workflow_run: workflows: - - baseline + - benchmark types: - completed branches: @@ -31,13 +31,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.repository.default_branch }} - - name: download baseline results + - name: download evaluation results env: GH_TOKEN: ${{ github.token }} run: | gh api --paginate \ "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.expired == false and (.name | test("^baseline-[0-9]+$"))) | [.created_at, .id] | @tsv' | + --jq '.artifacts[] | select(.expired == false and (.name | test("^benchmark-[0-9]+$"))) | [.created_at, .id] | @tsv' | sort | cut -f2 | while read -r artifact_id; do diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 75ef944f..43cb8956 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -27,7 +27,17 @@ permissions: contents: read jobs: + setup: + runs-on: ubuntu-latest + outputs: + run-date: ${{ steps.run.outputs.date }} + steps: + - name: Set run date + id: run + run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" + run: + needs: setup runs-on: ubuntu-latest strategy: fail-fast: true @@ -53,33 +63,38 @@ jobs: CONCURRENCY: ${{ inputs.concurrency }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} EXPERIMENT_NAME: ${{ inputs.experiment }} - OUTPUT_FILE: output-${{ matrix.order }}.json + RUN_DATE: ${{ needs.setup.outputs.run-date }} SHARD: ${{ matrix.order }}/${{ strategy.job-total }} run: | set -o pipefail + run_directory="results/experiments/$EXPERIMENT_NAME/$RUN_DATE" node packages/agent-eval/dist/cli.js \ --concurrency "$CONCURRENCY" \ --docker-image "$DOCKER_IMAGE" \ --experiment "$EXPERIMENT_NAME" \ --experiments experiments \ --scenarios scenarios \ - --artifacts artifacts \ - --output "$OUTPUT_FILE" \ + --output "$run_directory/output-${{ matrix.order }}.json" \ --shard "$SHARD" + - name: Prepare experiment artifact + if: ${{ always() }} + run: | + mkdir -p workflow-artifact + if [[ -d results ]]; then + mv results workflow-artifact/results + fi - name: Upload experiment results if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: experiment-shard-${{ github.run_id }}-${{ matrix.order }} - path: | - output-${{ matrix.order }}.json - artifacts + path: workflow-artifact if-no-files-found: error retention-days: 1 compression-level: 9 merge: - needs: run + needs: [setup, run] runs-on: ubuntu-latest steps: - name: download experiment results @@ -89,27 +104,25 @@ jobs: merge-multiple: true - name: merge experiment results run: | + cd "results/experiments/${{ inputs.experiment }}/${{ needs.setup.outputs.run-date }}" jq --slurp ' - . as $outputs - | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments - | $outputs[0] - | .treatments = $treatments - | .results = [ - $outputs[] as $output - | $output.results[] - | . as $result - | ($output.treatments[] | select(.id == $result.treatmentId).config.name) as $name - | .treatmentId = ($treatments[] | select(.config.name == $name).id) - ] + { + experimentId: .[0].experimentId, + scenarios: (reduce .[].scenarios as $scenarios ({}; . * $scenarios)), + treatments: (reduce .[].treatments as $treatments ({}; . * $treatments)), + trials: (reduce .[].trials as $trials ({}; . * $trials)) + } ' output-*.json > output.json rm output-*.json + - name: Prepare experiment artifact + run: | + mkdir -p workflow-artifact + mv results workflow-artifact/results - name: upload experiment results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: experiment - path: | - output.json - artifacts + path: workflow-artifact if-no-files-found: error retention-days: 90 compression-level: 9 diff --git a/.github/workflows/report-workflow-failures.yml b/.github/workflows/report-workflow-failures.yml index 4ed8f179..c24d1861 100644 --- a/.github/workflows/report-workflow-failures.yml +++ b/.github/workflows/report-workflow-failures.yml @@ -3,7 +3,7 @@ name: Report workflow failures on: workflow_run: workflows: - - baseline + - benchmark - deploy - update-pnpm - update-sandbox-tools diff --git a/.gitignore b/.gitignore index 83298f2e..b0ba3844 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ out # Generated files artifacts/ results/ +output.json diff --git a/README.md b/README.md index f624b4d9..df58d500 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ ## What this project does This project provides a framework for using agent-eval to evaluate scenarios. -Each experiment defines treatments that set up the agent's environment before -it completes one or more scenarios. +Experiments compare treatments across selected scenarios, while benchmarks +group scenarios into capabilities to measure broader agent performance. Scenarios represent tasks where, given a prompt, we measure how the agent behaves. This framework helps compare the effectiveness of different treatments @@ -22,12 +22,14 @@ Results are scored by: - **Scenarios** describe the task and tests used to grade the agent's output. - **Experiments** select the models, scenarios, and treatments to run together. +- **Benchmarks** group scenarios by capability and compare them against the + control treatment. - **Treatments** define the conditions for a run, such as adding an MCP server, custom sub-agent, or skill before the scenario starts. -## Running experiments +## Running evaluations -Run local experiments with the `agent-eval` CLI: +Run a local experiment with the `agent-eval` CLI: ```sh COPILOT_GITHUB_TOKEN=... agent-eval \ @@ -36,30 +38,35 @@ COPILOT_GITHUB_TOKEN=... agent-eval \ --experiment mcp ``` -You can also provide a path directly to `--experiment`: +Run a benchmark by selecting a file from the benchmarks directory: ```sh COPILOT_GITHUB_TOKEN=... agent-eval \ - --experiment ./experiments/mcp.ts \ + --benchmarks ./benchmarks \ + --benchmark design-system \ --scenarios ./scenarios ``` -The experiment and scenario directories default to `./experiments` and -`./scenarios`, respectively. Results are written to `./output.json` by default. -The output includes a run ID, the selected experiment ID, and an array of -treatment results. +The benchmark, experiment, and scenario directories default to `./benchmarks`, +`./experiments`, and `./scenarios`. Results are written to `./output.json` by +default, with trial artifacts stored in `./artifacts`. + +Use `--output-dir ` to keep `output.json` and its artifacts together +with portable relative paths. It cannot be combined with `--output` or +`--artifacts`. ## Authoring scenarios Scenarios live in [`./scenarios`](./scenarios/). Each scenario has a `scenario.config.ts` file that defines the agent prompt and a `scenario.test.ts` file that grades the agent's output. Scenarios can also include an optional -`scenario.browser.test.ts` file for checks that need a browser: +`browser.test.ts` file for checks that need a browser. The legacy +`scenario.browser.test.ts` filename remains supported: ```ts -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent uses a Primer button correctly', prompt: 'Update the index page to use a primary button', tags: ['baseline', 'button', 'primer'], @@ -100,10 +107,8 @@ export const experiment = defineConfig({ }) ``` -Each model config has a `name` and a `reasoningEfforts` array. The experiment -runs once for each configured effort. Model information, including each model's -supported reasoning efforts, is exported as `models` from -`@primer/agent-eval`. +Models can be specified by name to use the default `medium` reasoning effort or +with a `name` and `reasoningEfforts` array to run multiple variants. The experiment config specifies: @@ -138,8 +143,9 @@ export const experiment = defineConfig({ Scenarios can also be defined inline in an experiment. Inline scenario paths resolve from the directory where the CLI is run, and use the same -`scenario.config.ts`, `scenario.test.ts`, and optional `scenario.browser.test.ts` -files as repository scenarios: +`scenario.config.ts`, `scenario.test.ts`, and optional browser test files as +repository scenarios. Use `name` to override the scenario ID derived from the +directory name: ```ts export const experiment = defineConfig({ @@ -148,6 +154,7 @@ export const experiment = defineConfig({ models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], scenarios: [ { + name: 'local-button', path: './scenarios/local-button-scenario', }, ], @@ -155,7 +162,31 @@ export const experiment = defineConfig({ }) ``` -### Describe eval tests +## Authoring benchmarks + +Benchmarks live in [`./benchmarks`](./benchmarks/) and group scenarios into +capabilities: + +```ts +import {defineConfig} from '@primer/agent-eval/benchmark' + +export const benchmark = defineConfig({ + name: 'Design system', + description: 'Measure agent performance across design system tasks', + models: ['gpt-5.6-sol'], + capabilities: [ + { + name: 'Uses components', + scenarios: ['001-agent-uses-button-from-primer'], + }, + ], +}) +``` + +Each benchmark runs the control treatment for every configured model variant +and scenario. + +## Describing eval tests Individual test titles and statuses are included in each run's `testResults`. Add a JSDoc, block, or consecutive line comment immediately before a test to diff --git a/benchmarks/design-system.ts b/benchmarks/design-system.ts new file mode 100644 index 00000000..1e4422a5 --- /dev/null +++ b/benchmarks/design-system.ts @@ -0,0 +1,120 @@ +import {defineConfig} from '@primer/agent-eval/benchmark' + +export const benchmark = defineConfig({ + name: 'Design System', + description: 'Benchmark the performance of agents with different design system tasks.', + models: ['gpt-5.6-sol', 'gpt-5.6-terra', 'claude-opus-5', 'claude-sonnet-5', 'gemini-3.5-flash'], + async setup({sandbox}) { + // Setup the Primer MCP server locally + await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest']) + await sandbox.addMcpServer('primer', { + type: 'local', + command: 'npx', + args: ['--no-install', '@primer/mcp'], + tools: ['*'], + }) + }, + capabilities: [ + { + name: 'Uses appropriate components by default', + scenarios: ['001-agent-uses-button-from-primer', '003-agent-uses-form-from-primer'], + }, + { + name: 'Uses Octicons by default for iconography', + scenarios: ['002-agent-uses-octicon-from-primer'], + }, + { + name: 'Sets up new projects with Primer', + scenarios: ['004-agent-setup-nextjs', '005-agent-enables-theme-switching'], + }, + // { + // name: 'Infers correct component based on usage', + // scenarios: ['007-agent-infers-billing-banner', '008-agent-infers-action-menu'], + // }, + // { + // name: 'Composes components correctly', + // scenarios: [], + // }, + // { + // name: 'Uses documented component APIs', + // scenarios: [], + // }, + // { + // name: 'Uses tokens', + // scenarios: [ + // '009-agent-uses-layout-and-color-tokens', + // '010-agent-uses-typography-tokens', + // '011-agent-uses-motion-tokens', + // ], + // }, + // { + // name: 'Infers correct token based on usage', + // scenarios: ['012-agent-infers-status-tokens', '013-agent-infers-compact-control-tokens'], + // }, + // { + // name: 'Uses icons', + // scenarios: ['002-agent-uses-octicon-from-primer', '014-agent-replaces-custom-icons-with-octicons'], + // }, + // { + // name: 'Infers correct icon based on usage', + // scenarios: ['015-agent-infers-copy-icon'], + // }, + // { + // name: 'Uses UI patterns', + // scenarios: [ + // '003-agent-uses-form-from-primer', + // '016-agent-uses-loading-and-empty-state-patterns', + // '017-agent-uses-confirmation-pattern', + // '018-agent-uses-filter-pattern', + // ], + // }, + // { + // name: 'Implements navigation patterns', + // scenarios: [], + // }, + // { + // name: 'Applies accessibility guidance', + // scenarios: [], + // }, + // { + // name: 'Builds responsive interfaces', + // scenarios: [], + // }, + // { + // name: 'Handles interaction states', + // scenarios: [], + // }, + // { + // name: 'Uses utilities', + // scenarios: ['019-agent-uses-dismissal-utilities', '020-agent-uses-resize-observer-utility'], + // }, + // { + // name: 'Extends the design system safely', + // scenarios: [], + // }, + // { + // name: 'Getting started', + // scenarios: ['004-agent-setup-nextjs', '021-agent-sets-up-primer-in-vite'], + // }, + // { + // name: 'Theming', + // scenarios: [ + // '005-agent-enables-theme-switching', + // '022-agent-enables-automatic-theming', + // '023-agent-adds-theme-switcher', + // ], + // }, + // { + // name: 'Supports accessible color modes', + // scenarios: [], + // }, + // { + // name: 'Works with TailwindCSS', + // scenarios: ['024-agent-sets-up-tailwindcss', '025-agent-uses-tokens-with-tailwindcss'], + // }, + // { + // name: 'Respects component maturity', + // scenarios: ['026-agent-avoids-deprecated-notification'], + // }, + ], +}) diff --git a/benchmarks/noop.ts b/benchmarks/noop.ts new file mode 100644 index 00000000..1e7fbde5 --- /dev/null +++ b/benchmarks/noop.ts @@ -0,0 +1,24 @@ +import {defineConfig} from '@primer/agent-eval/benchmark' + +export default defineConfig({ + name: 'noop', + description: 'noop benchmark for end-to-end testing', + models: [ + { + name: 'gpt-5.6-luna', + reasoningEfforts: ['low'], + }, + ], + async setup() { + console.log('global setup') + }, + capabilities: [ + { + name: 'noop', + scenarios: ['001-agent-uses-button-from-primer'], + async setup() { + console.log('local setup') + }, + }, + ], +}) diff --git a/benchmarks/package.json b/benchmarks/package.json new file mode 100644 index 00000000..f93643c7 --- /dev/null +++ b/benchmarks/package.json @@ -0,0 +1,13 @@ +{ + "name": "@primer/benchmarks", + "private": true, + "type": "module", + "scripts": { + "type-check": "tsc --noEmit" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/node": "^26.1.2", + "typescript": "^6.0.3" + } +} diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json new file mode 100644 index 00000000..df8a02ab --- /dev/null +++ b/benchmarks/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig.base.json", + "include": ["*.ts"] +} + diff --git a/experiments/baseline.ts b/experiments/baseline.ts deleted file mode 100644 index 40d440ca..00000000 --- a/experiments/baseline.ts +++ /dev/null @@ -1,37 +0,0 @@ -import path from 'node:path' -import {defineConfig} from '@primer/agent-eval/experiment' -import {listScenarios} from '@primer/agent-eval' - -const scenarios = await listScenarios({ - directory: path.resolve(import.meta.dirname, '..', 'scenarios'), - tags: ['baseline'], -}) - -export const experiment = defineConfig({ - name: 'Baseline', - description: 'Baseline experiment to evaluate the performance of the agent with our recommended setup.', - models: [ - {name: 'gpt-5.6-sol', reasoningEfforts: []}, - {name: 'gpt-5.6-terra', reasoningEfforts: []}, - {name: 'claude-opus-5', reasoningEfforts: []}, - {name: 'claude-sonnet-5', reasoningEfforts: []}, - {name: 'gemini-3.1-pro-preview', reasoningEfforts: []}, - {name: 'gemini-3.6-flash', reasoningEfforts: []}, - ], - scenarios: scenarios.filter(scenario => !scenario.id.startsWith('000')).map(scenario => scenario.id), - treatments: [ - { - name: 'Recommended', - async setup({sandbox}) { - // Setup the Primer MCP server locally - await sandbox.runCommand('npm', ['install', '-g', '@primer/mcp@latest']) - await sandbox.addMcpServer('primer', { - type: 'local', - command: 'npx', - args: ['--no-install', '@primer/mcp'], - tools: ['*'], - }) - }, - }, - ], -}) diff --git a/experiments/noop.ts b/experiments/noop.ts index 30cf76e4..e7aa5198 100644 --- a/experiments/noop.ts +++ b/experiments/noop.ts @@ -10,5 +10,15 @@ export const experiment = defineConfig({ }, ], scenarios: ['001-agent-uses-button-from-primer'], - treatments: [], + async setup() { + console.log('global setup') + }, + treatments: [ + { + name: 'noop', + async setup() { + console.log('local setup') + }, + }, + ], }) diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 3556a3f5..bd02ff20 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -1,7 +1,7 @@ # @primer/agent-eval -A library and cli tool for creating and running experiments in order to evaluate -agent behavior across different scenarios. +A library and CLI tool for creating and running experiments and benchmarks that +evaluate agent behavior across different scenarios. ## Getting started @@ -12,7 +12,7 @@ command using [npm](https://www.npmjs.com/): npm install -S @primer/agent-eval ``` -This will provide both the cli and library for creating and running experiments. +This provides the `agent-eval` executable and the package's programmatic APIs. Typically, you'll first create an experiment: ```tsx @@ -71,9 +71,9 @@ against: ```tsx // scenarios/uses-button-from-primer/scenario.config.ts -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent completes the example task', prompt: `Example scenario prompt that will instruct the agent to perform a task`, tags: ['baseline', 'button', 'primer'], @@ -96,13 +96,12 @@ agent sees the prompt for the scenario. ### Browser tests -Add an optional `scenario.browser.test.ts` file when a scenario needs tests in a -real browser. Agent eval installs Vitest, Playwright, Vitest's Playwright browser -provider, Chromium, and its system dependencies, then runs the browser test in -Vitest browser mode after the required `scenario.test.ts` file. Results from -both files are combined in the scenario score and test-results artifact. +Add an optional `browser.test.ts` file when a scenario needs tests in a real +browser. The legacy `scenario.browser.test.ts` filename remains supported. +Agent eval runs browser tests with Playwright after `scenario.test.ts` and +combines both results in the scenario score and test-results artifact. -With everything in place, you can now use the `@primer/agent-eval` cli to run +With everything in place, you can now use the `agent-eval` executable to run the experiment: ```bash @@ -123,20 +122,52 @@ COPILOT_GITHUB_TOKEN=... agent-eval \ Use `--experiments` to load experiment files from a local directory. Experiment files may export an `experiment` named export or a default export. `--experiment` -may also be a path to a local experiment file when you only want to run one -experiment. The experiments directory defaults to `./experiments`. Use -`--scenarios` to set the directory containing scenario directories; it defaults -to `./scenarios`. +selects an experiment by its filename without the extension. The experiments +directory defaults to `./experiments`. Use `--scenarios` to set the directory +containing scenario directories; it defaults to `./scenarios`. + +Use `--benchmark` to select a benchmark by filename and `--benchmarks` to set +the benchmark directory: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval \ + --benchmarks ./benchmarks \ + --scenarios ./scenarios \ + --benchmark design-system +``` + +### Result bundles + +Keep the output file and artifacts in one directory so results can be moved +between machines without rewriting paths: + +```text +run/ +├── output.json +└── artifacts/ +``` + +```sh +agent-eval \ + --experiment example \ + --output-dir run +``` + +Artifact and walkthrough references written by the CLI are relative to the +directory containing `output.json`. Upload or download the complete `run` +directory to preserve those references. `--output-dir` creates `output.json` +and `artifacts/` within the selected directory. When using `--output`, artifacts +are written to an `artifacts/` directory beside the selected file. ## Scenario config authoring -Use `defineScenario` from `@primer/agent-eval/scenario` in each +Use `defineConfig` from `@primer/agent-eval/scenario` in each `scenario.config.ts` file: ```ts -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent uses a Primer button correctly', prompt: 'Update the index page to use a primary button', tags: ['baseline', 'button', 'primer'], @@ -144,8 +175,7 @@ export default defineScenario({ ``` Scenario descriptions and tags are optional. Use `description` to explain what -the scenario tests. Pass `tags` to `listScenarios` to return only scenarios that -include every requested tag. +the scenario tests. ## Experiment config authoring @@ -164,10 +194,51 @@ export const experiment = defineConfig({ }) ``` -Each model config has a `name` and a `reasoningEfforts` array. The experiment -runs once for each configured effort. Model information, including each model's -supported reasoning efforts, is exported as `models` from -`@primer/agent-eval`. +Models can be specified by name to use the default `medium` reasoning effort or +with a `name` and `reasoningEfforts` array to run multiple variants. + +Scenarios can be selected by ID or loaded directly from a path: + +```ts +scenarios: [ + '001-agent-uses-button-from-primer', + { + name: 'local-button', + path: './scenarios/local-button-scenario', + }, +] +``` + +## Benchmark config authoring + +Use `defineConfig` from `@primer/agent-eval/benchmark` to group scenarios into +capabilities: + +```ts +import {defineConfig} from '@primer/agent-eval/benchmark' + +export const benchmark = defineConfig({ + name: 'Design system', + description: 'Measure agent performance across design system tasks', + models: ['gpt-5.6-sol'], + async setup({sandbox}) { + await sandbox.addAgentSkill('design-system', 'Uses the design system', 'Follow the design system guidance.') + }, + capabilities: [ + { + name: 'Uses components', + scenarios: ['001-agent-uses-button-from-primer'], + async setup({sandbox}) { + await sandbox.writeFile('/root/.copilot/component-guidance.md', 'Prefer existing components.') + }, + }, + ], +}) +``` + +The top-level setup runs first for every benchmark treatment trial. A +capability setup runs next for treatment trials in that capability. Control +trials do not run either setup. Treatment setup can add custom Copilot sub-agents to `~/.copilot/agents`: @@ -224,3 +295,18 @@ await sandbox.addCopilotPlugin({ Use `{type: 'local', sourcePath: './plugins/local-marketplace'}` as the marketplace `source` to install from a local marketplace. + +## Programmatic APIs + +The package root exports explicitly named benchmark, experiment, scenario, +treatment, and trial APIs. Domain entry points are available from +`@primer/agent-eval/benchmark`, `@primer/agent-eval/experiment`, +`@primer/agent-eval/scenario`, and `@primer/agent-eval/sandbox`. + +Use the benchmark and experiment entry points for configuration, discovery, +execution, output creation, serialization, and deserialization. Use the +sandbox entry point for `Sandbox`, `SystemSandbox`, `VirtualSandbox`, plugin and +MCP configuration types, and sandbox constants. + +The CLI is available through the `agent-eval` executable rather than a +`@primer/agent-eval/cli` package entry point. diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index 673cc8d7..0344e8a5 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -10,29 +10,21 @@ "types": "./dist/index.d.ts", "default": "./dist/index.js" }, - "./cli": { - "types": "./dist/cli.d.ts", - "default": "./dist/cli.js" + "./benchmark": { + "types": "./dist/benchmark.d.ts", + "default": "./dist/benchmark.js" }, "./experiment": { "types": "./dist/experiment.d.ts", "default": "./dist/experiment.js" }, - "./experiments": { - "types": "./dist/experiments.d.ts", - "default": "./dist/experiments.js" - }, - "./output": { - "types": "./dist/output.d.ts", - "default": "./dist/output.js" - }, "./scenario": { "types": "./dist/scenario.d.ts", "default": "./dist/scenario.js" }, - "./scenarios": { - "types": "./dist/scenarios.d.ts", - "default": "./dist/scenarios.js" + "./sandbox": { + "types": "./dist/sandbox/index.d.ts", + "default": "./dist/sandbox/index.js" } }, "engines": { @@ -59,6 +51,9 @@ }, "dependencies": { "dockerode": "^5.0.1", + "p-queue": "^9.3.3", + "pino": "^10.3.1", + "pino-pretty": "^13.1.3", "tar-fs": "^3.1.2", "tar-stream": "^3.2.0", "zod": "^4.4.3" @@ -68,6 +63,7 @@ "@types/node": "^26.4.0", "@types/tar-fs": "^2.0.4", "@types/tar-stream": "^3.1.4", + "memfs": "^4.68.1", "publint": "^0.3.24", "rimraf": "^6.1.3", "rolldown": "^1.2.6", diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index c5ddf713..7e1d288b 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -9,10 +9,11 @@ const external = dependencies.map(name => { const config = defineConfig({ input: { + benchmark: 'src/benchmark.ts', cli: 'src/cli.ts', experiment: 'src/experiment.ts', index: 'src/index.ts', - scenario: 'src/scenario-config.ts', + scenario: 'src/scenario.ts', }, platform: 'node', external, diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts new file mode 100644 index 00000000..6d9e74ce --- /dev/null +++ b/packages/agent-eval/src/benchmark.test.ts @@ -0,0 +1,402 @@ +import {afterEach, expect, test, vi} from 'vitest' +import { + defineConfig, + deserialize, + getBenchmark, + listBenchmarks, + output, + run, + serialize, + type BenchmarkTrialResult, +} from './benchmark' +import {VirtualHost} from './host' +import {run as runPlan} from './plan' +import {defineConfig as defineScenarioConfig} from './scenario' + +vi.mock('./plan', async importOriginal => { + const original = await importOriginal() + return { + ...original, + run: vi.fn(original.run), + } +}) + +afterEach(() => { + vi.clearAllMocks() + vi.restoreAllMocks() +}) + +const config = defineConfig({ + name: 'Test benchmark', + description: 'Tests a benchmark', + models: ['gpt-5.6-sol'], + capabilities: [ + { + name: 'Test capability', + scenarios: ['001-scenario'], + }, + ], +}) + +const scenario = defineScenarioConfig({ + prompt: 'Complete the task', +}) + +function createHost(files: Record) { + return VirtualHost.create({ + '/benchmarks': files, + '/scenarios': { + '001-scenario': { + 'scenario.config.ts': `export default ${JSON.stringify(scenario)}`, + 'scenario.test.ts': '', + 'package.json': JSON.stringify({}), + }, + }, + }) +} + +test('listBenchmarks loads configs and resolves models and capability scenarios', async () => { + const serializedConfig = JSON.stringify(config) + const host = createHost({ + 'named.ts': `export const benchmark = ${serializedConfig}`, + 'default.js': `export default ${serializedConfig}`, + 'types.d.ts': `export const benchmark = ${serializedConfig}`, + 'index.ts': `export const benchmark = ${serializedConfig}`, + 'unsupported.json': serializedConfig, + 'missing.ts': 'export const value = true', + 'invalid.ts': 'export const benchmark = {}', + }) + + const benchmarks = await listBenchmarks({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }) + + expect(benchmarks).toHaveLength(2) + expect(benchmarks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'named', + filepath: '/benchmarks/named.ts', + name: config.name, + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + capabilities: [ + { + name: 'Test capability', + scenarios: [ + expect.objectContaining({ + id: '001-scenario', + prompt: 'Complete the task', + }), + ], + }, + ], + }), + expect.objectContaining({ + id: 'default', + filepath: '/benchmarks/default.js', + }), + ]), + ) +}) + +test('listBenchmarks prefers the named benchmark export', async () => { + const namedConfig = { + ...config, + name: 'Named benchmark', + } + const defaultConfig = { + ...config, + name: 'Default benchmark', + } + const host = createHost({ + 'benchmark.ts': [ + `export const benchmark = ${JSON.stringify(namedConfig)}`, + `export default ${JSON.stringify(defaultConfig)}`, + ].join('\n'), + }) + + await expect( + listBenchmarks({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }), + ).resolves.toEqual([ + expect.objectContaining({ + id: 'benchmark', + name: 'Named benchmark', + }), + ]) +}) + +test('listBenchmarks sorts benchmarks by filename', async () => { + const serializedConfig = JSON.stringify(config) + const host = createHost({ + 'z-last.ts': `export const benchmark = ${serializedConfig}`, + 'a-first.ts': `export const benchmark = ${serializedConfig}`, + }) + + const benchmarks = await listBenchmarks({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }) + + expect( + benchmarks.map(benchmark => { + return benchmark.id + }), + ).toEqual(['a-first', 'z-last']) +}) + +test('listBenchmarks validates the benchmarks directory', async () => { + const host = VirtualHost.create() + + await expect( + listBenchmarks({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }), + ).rejects.toThrow('Benchmarks directory does not exist: /benchmarks') + + const fileHost = VirtualHost.create({ + '/benchmarks': '', + }) + + await expect( + listBenchmarks({ + host: fileHost, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + }), + ).rejects.toThrow('Benchmarks path is not a directory: /benchmarks') +}) + +test('getBenchmark returns the benchmark matching the id', async () => { + const serializedConfig = JSON.stringify(config) + const host = createHost({ + 'first.ts': `export const benchmark = ${serializedConfig}`, + 'second.ts': `export const benchmark = ${serializedConfig}`, + }) + + await expect( + getBenchmark({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + id: 'second', + }), + ).resolves.toEqual( + expect.objectContaining({ + id: 'second', + filepath: '/benchmarks/second.ts', + name: config.name, + }), + ) +}) + +test('getBenchmark throws when the benchmark is not found', async () => { + const host = createHost({}) + + await expect( + getBenchmark({ + host, + benchmarksDirectory: '/benchmarks', + scenariosDirectory: '/scenarios', + id: 'missing', + }), + ).rejects.toThrow('Benchmark "missing" was not found in: /benchmarks') +}) + +test('run returns an empty result when the benchmark has no trials', async () => { + const emptyConfig = defineConfig({ + name: 'Empty benchmark', + description: 'Has no trials', + models: [], + capabilities: [], + }) + const host = createHost({ + 'empty.ts': `export const benchmark = ${JSON.stringify(emptyConfig)}`, + }) + + await expect( + run({ + env: { + artifactsDirectory: '/artifacts', + benchmarksDirectory: '/benchmarks', + concurrency: 1, + copilotToken: 'token', + dockerImage: 'node:26-slim', + experimentsDirectory: '/experiments', + outputPath: '/output.json', + scenariosDirectory: '/scenarios', + }, + host, + id: 'empty', + }), + ).resolves.toEqual([]) +}) + +test('run applies global and capability setup to benchmark treatment trials', async () => { + const setupOrder: Array = [] + const benchmarkConfig = defineConfig({ + name: 'Benchmark with setup', + description: 'Runs global and capability setup', + models: ['gpt-5.6-sol'], + async setup() { + setupOrder.push('global') + }, + capabilities: [ + { + name: 'Capability with setup', + scenarios: ['001-scenario'], + async setup() { + setupOrder.push('capability') + }, + }, + ], + }) + const host = createHost({ + 'with-setup.ts': '', + }) + const loadModule = host.loadModule.bind(host) + vi.spyOn(host, 'loadModule').mockImplementation(async filepath => { + if (filepath === '/benchmarks/with-setup.ts') { + return { + benchmark: benchmarkConfig, + } + } + + return loadModule(filepath) + }) + vi.mocked(runPlan).mockImplementationOnce(async ({plan}) => { + const controlTrial = plan.trials.find(trial => { + return trial.treatment.name === 'Control' + }) + const benchmarkTrial = plan.trials.find(trial => { + return trial.treatment.name === 'Benchmark' + }) + + expect(controlTrial?.treatment.setup).toBeUndefined() + expect(benchmarkTrial?.treatment.setup).toBeDefined() + + const sandbox = await host.createSandbox() + await benchmarkTrial?.treatment.setup?.({sandbox}) + + return [] + }) + + await expect( + run({ + env: { + artifactsDirectory: '/artifacts', + benchmarksDirectory: '/benchmarks', + concurrency: 1, + copilotToken: 'token', + dockerImage: 'node:26-slim', + experimentsDirectory: '/experiments', + outputPath: '/output.json', + scenariosDirectory: '/scenarios', + }, + host, + id: 'with-setup', + }), + ).resolves.toEqual([]) + + expect(setupOrder).toEqual(['global', 'capability']) +}) + +test('output serializes and deserializes benchmark capability metadata', () => { + const capability = { + name: 'Test capability', + scenarios: [ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }, + ], + } + const trialResult: BenchmarkTrialResult = { + capability, + artifacts: { + directory: '/bundle/artifacts/trial', + copilotConfigDirectory: '/bundle/artifacts/trial/.copilot', + skillsConfigDirectory: '/bundle/artifacts/trial/.agents', + testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json', + workspaceDirectory: '/bundle/artifacts/trial/workspace', + }, + trial: { + id: 'trial', + scenario: capability.scenarios[0], + treatment: { + name: 'Benchmark', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + }, + agent: { + sessions: [], + }, + testResults: { + numTotalTests: 1, + numPassedTests: 1, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }, + walkthrough: { + type: 'Screenshots', + screenshots: [ + '/bundle/artifacts/trial/walkthrough/screenshots/01.png', + '/bundle/artifacts/trial/walkthrough/screenshots/02.png', + ], + }, + } + + const benchmarkOutput = output('test-benchmark', [trialResult], { + baseDirectory: '/bundle', + }) + + expect(benchmarkOutput.benchmarkId).toBe('test-benchmark') + expect(benchmarkOutput.capabilities.get('Test capability')).toEqual({ + name: 'Test capability', + scenarioIds: ['001-scenario'], + }) + expect(benchmarkOutput.trials.get('trial')).toEqual( + expect.objectContaining({ + capabilityId: 'Test capability', + artifacts: { + directory: 'artifacts/trial', + copilotConfigDirectory: 'artifacts/trial/.copilot', + skillsConfigDirectory: 'artifacts/trial/.agents', + testResultsPath: 'artifacts/trial/workspace/test-results.json', + workspaceDirectory: 'artifacts/trial/workspace', + }, + scenarioId: '001-scenario', + treatmentId: 'Benchmark', + walkthrough: { + type: 'Screenshots', + screenshots: [ + 'artifacts/trial/walkthrough/screenshots/01.png', + 'artifacts/trial/walkthrough/screenshots/02.png', + ], + }, + }), + ) + expect(deserialize(serialize(benchmarkOutput))).toEqual(benchmarkOutput) +}) diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts new file mode 100644 index 00000000..fafd9ecf --- /dev/null +++ b/packages/agent-eval/src/benchmark.ts @@ -0,0 +1,377 @@ +import {randomUUID} from 'node:crypto' +import path from 'node:path' +import * as z from 'zod/mini' +import type {EnvironmentConfig} from './environment' +import {DefaultHost, type Host} from './host' +import {logger} from './logger' +import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model' +import {create as createPlan, run as runPlan} from './plan' +import {getScenario, ScenarioSchema, type Scenario} from './scenario' +import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' +import { + getPortableTrialPaths, + TrialAgentSchema, + TrialArtifactsSchema, + WalkthroughSchema, + type Trial, + type TrialResult, +} from './trial' +import {TestResultsSchema} from './vitest' + +const CapabilityConfigSchema = z.object({ + name: z.string(), + scenarios: z.array(z.string()), + setup: z.optional(TreatmentSetupSchema), +}) + +const BenchmarkConfigSchema = z.object({ + name: z.string(), + description: z.string(), + models: ModelVariantConfigSchema, + setup: z.optional(TreatmentSetupSchema), + capabilities: z.array(CapabilityConfigSchema), +}) + +type BenchmarkConfig = z.infer + +function defineConfig(config: Config): Config { + return config +} + +type Capability = { + name: string + scenarios: Array + setup?: TreatmentSetup +} + +type Benchmark = { + id: string + filepath: string + name: BenchmarkConfig['name'] + description: BenchmarkConfig['description'] + models: Array + setup?: TreatmentSetup + capabilities: Array +} + +type BenchmarkModule = { + benchmark?: unknown + default?: unknown +} + +const BENCHMARK_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts']) + +async function listBenchmarks({ + host = DefaultHost, + benchmarksDirectory, + scenariosDirectory, +}: { + host?: Host + benchmarksDirectory: string + scenariosDirectory: string +}): Promise> { + if (!host.existsSync(benchmarksDirectory)) { + throw new Error(`Benchmarks directory does not exist: ${benchmarksDirectory}`) + } + + const stats = await host.fs.stat(benchmarksDirectory) + if (!stats.isDirectory()) { + throw new Error(`Benchmarks path is not a directory: ${benchmarksDirectory}`) + } + + const filenames = (await host.fs.readdir(benchmarksDirectory)).sort() + const benchmarks: Array = [] + + for (const filename of filenames) { + if (!isBenchmarkFile(filename)) { + continue + } + + const filepath = path.join(benchmarksDirectory, filename) + const mod: BenchmarkModule = await host.loadModule(filepath) + const data = mod.benchmark ?? mod.default + if (!data) { + continue + } + + const parseResult = BenchmarkConfigSchema.safeParse(data) + if (!parseResult.success) { + logger.warn( + `Failed to parse benchmark config for file: ${filepath}. Error: ${z.prettifyError(parseResult.error)}`, + ) + continue + } + + const {data: config} = parseResult + const capabilities = await Promise.all( + config.capabilities.map(async capability => { + const scenarios = await Promise.all( + capability.scenarios.map(scenario => { + return getScenario(host, scenariosDirectory, scenario) + }), + ) + + return { + name: capability.name, + scenarios, + setup: capability.setup, + } + }), + ) + + benchmarks.push({ + id: getBenchmarkId(filename), + filepath, + name: config.name, + description: config.description, + models: getModelVariants(config.models), + setup: config.setup, + capabilities, + }) + } + + return benchmarks +} + +async function getBenchmark({ + host = DefaultHost, + benchmarksDirectory, + scenariosDirectory, + id, +}: { + host?: Host + benchmarksDirectory: string + scenariosDirectory: string + id: string +}): Promise { + const benchmarks = await listBenchmarks({ + host, + benchmarksDirectory, + scenariosDirectory, + }) + const benchmark = benchmarks.find(candidate => candidate.id === id) + if (benchmark) { + return benchmark + } + + throw new Error(`Benchmark "${id}" was not found in: ${benchmarksDirectory}`) +} + +function getBenchmarkId(filename: string): string { + return path.basename(filename, path.extname(filename)) +} + +function isBenchmarkFile(filename: string): boolean { + return !filename.endsWith('.d.ts') && filename !== 'index.ts' && BENCHMARK_FILE_EXTENSIONS.has(path.extname(filename)) +} + +type BenchmarkTrialResult = TrialResult & { + capability: Capability +} + +type BenchmarkRunResult = Array + +async function run({ + env, + host = DefaultHost, + id, +}: { + env: EnvironmentConfig + host?: Host + id: string +}): Promise { + const benchmark = await getBenchmark({ + host, + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id, + }) + const trialCapabilities = new Map() + const trials: Array = benchmark.models.flatMap(model => { + return benchmark.capabilities.flatMap(capability => { + const benchmarkTreatment = createBenchmarkTreatment(benchmark, capability) + return capability.scenarios.flatMap(scenario => { + return [ControlTreatment, benchmarkTreatment].map(treatment => { + const trial = { + id: randomUUID(), + scenario, + treatment, + model, + } + trialCapabilities.set(trial.id, capability) + return trial + }) + }) + }) + }) + const plan = await createPlan(trials) + const results = await runPlan({ + env, + host, + plan, + }) + + return results.map(result => { + const capability = trialCapabilities.get(result.trial.id) + if (!capability) { + throw new Error(`Capability was not found for trial: ${result.trial.id}`) + } + + return { + ...result, + capability, + } + }) +} + +function createBenchmarkTreatment(benchmark: Benchmark, capability: Capability): Treatment { + const setup = + benchmark.setup || capability.setup + ? async ({sandbox}: Parameters[0]) => { + await benchmark.setup?.({sandbox}) + await capability.setup?.({sandbox}) + } + : undefined + + return { + name: 'Benchmark', + setup, + } +} + +const CapabilityOutputSchema = z.object({ + name: z.string(), + scenarioIds: z.array(z.string()), +}) + +const BenchmarkTrialOutputSchema = z.object({ + agent: TrialAgentSchema, + artifacts: TrialArtifactsSchema, + capabilityId: z.string(), + id: z.string(), + model: ModelVariantSchema, + scenarioId: z.string(), + testResults: TestResultsSchema, + treatmentId: z.string(), + walkthrough: WalkthroughSchema, +}) + +const SerializedBenchmarkOutputSchema = z.object({ + benchmarkId: z.string(), + capabilities: z.record(z.string(), CapabilityOutputSchema), + scenarios: z.record( + z.string(), + z.pick(ScenarioSchema, { + id: true, + directory: true, + prompt: true, + description: true, + tags: true, + testPath: true, + browserTestPath: true, + }), + ), + treatments: z.record( + z.string(), + z.pick(TreatmentSchema, { + name: true, + }), + ), + trials: z.record(z.string(), BenchmarkTrialOutputSchema), +}) + +type BenchmarkOutput = { + benchmarkId: string + capabilities: Map> + scenarios: Map + treatments: Map + trials: Map> +} + +type BenchmarkOutputOptions = { + baseDirectory?: string +} + +function output( + benchmarkId: string, + trialResults: BenchmarkRunResult, + options: BenchmarkOutputOptions = {}, +): BenchmarkOutput { + const result: BenchmarkOutput = { + benchmarkId, + capabilities: new Map(), + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + } + + for (const trialResult of trialResults) { + const {capability, trial} = trialResult + const {artifacts, walkthrough} = options.baseDirectory + ? getPortableTrialPaths(trialResult, options.baseDirectory) + : trialResult + + if (!result.capabilities.has(capability.name)) { + result.capabilities.set(capability.name, { + name: capability.name, + scenarioIds: capability.scenarios.map(scenario => scenario.id), + }) + } + + if (!result.scenarios.has(trial.scenario.id)) { + result.scenarios.set(trial.scenario.id, trial.scenario) + } + + if (!result.treatments.has(trial.treatment.name)) { + result.treatments.set(trial.treatment.name, trial.treatment) + } + + result.trials.set(trial.id, { + agent: trialResult.agent, + artifacts, + capabilityId: capability.name, + id: trial.id, + model: trial.model, + scenarioId: trial.scenario.id, + testResults: trialResult.testResults, + treatmentId: trial.treatment.name, + walkthrough, + }) + } + + return result +} + +function serialize(benchmarkOutput: BenchmarkOutput): string { + return JSON.stringify({ + benchmarkId: benchmarkOutput.benchmarkId, + capabilities: Object.fromEntries(benchmarkOutput.capabilities), + scenarios: Object.fromEntries(benchmarkOutput.scenarios), + treatments: Object.fromEntries(benchmarkOutput.treatments), + trials: Object.fromEntries(benchmarkOutput.trials), + }) +} + +function deserialize(input: unknown): BenchmarkOutput { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + const result = SerializedBenchmarkOutputSchema.parse(parsed, {reportInput: true}) + + return { + benchmarkId: result.benchmarkId, + capabilities: new Map(Object.entries(result.capabilities)), + scenarios: new Map(Object.entries(result.scenarios)), + treatments: new Map(Object.entries(result.treatments)), + trials: new Map(Object.entries(result.trials)), + } +} + +export {BenchmarkConfigSchema, defineConfig, deserialize, getBenchmark, listBenchmarks, output, run, serialize} +export type { + BenchmarkConfig, + Benchmark, + BenchmarkOutput, + BenchmarkOutputOptions, + BenchmarkRunResult, + BenchmarkTrialResult, + Capability, +} diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts new file mode 100644 index 00000000..7f42fdba --- /dev/null +++ b/packages/agent-eval/src/cli.test.ts @@ -0,0 +1,55 @@ +import {afterEach, describe, expect, test, vi} from 'vitest' + +const originalArgv = process.argv +const originalToken = process.env.COPILOT_GITHUB_TOKEN + +afterEach(() => { + process.argv = originalArgv + if (originalToken === undefined) { + delete process.env.COPILOT_GITHUB_TOKEN + } else { + process.env.COPILOT_GITHUB_TOKEN = originalToken + } + vi.restoreAllMocks() + vi.resetModules() +}) + +describe('cli', () => { + test('displays help when requested', async () => { + process.argv = ['node', 'agent-eval', '--help'] + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.spyOn(process, 'exit').mockImplementation(() => { + throw new Error('process.exit') + }) + + await expect(import('./cli')).rejects.toThrow('process.exit') + expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]')) + expect(log).toHaveBeenCalledWith(expect.stringContaining('--output-dir ')) + }) + + test('requires a Copilot token before running', async () => { + process.argv = ['node', 'agent-eval'] + delete process.env.COPILOT_GITHUB_TOKEN + + await expect(import('./cli')).rejects.toThrow( + 'COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval', + ) + }) + + test('displays help when no benchmark or experiment is selected', async () => { + process.argv = ['node', 'agent-eval'] + process.env.COPILOT_GITHUB_TOKEN = 'token' + const log = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await import('./cli') + + expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]')) + }) + + test('rejects output directory combinations with explicit output paths', async () => { + process.argv = ['node', 'agent-eval', '--output-dir', 'results/run', '--output', 'output.json'] + process.env.COPILOT_GITHUB_TOKEN = 'token' + + await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --output') + }) +}) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index e56a7ff9..ed9d8501 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -1,24 +1,37 @@ #!/usr/bin/env node -import {randomUUID} from 'node:crypto' + import {existsSync} from 'node:fs' -import path from 'node:path' import fs from 'node:fs/promises' +import path from 'node:path' import {parseArgs} from 'node:util' -import {ControlTreatment, type ExperimentConfig} from './experiment-config' -import {resolveModelConfigs, type Model, type ReasoningEffort} from './model' -import {createAgentEvalOutput} from './output' -import type {Treatment, TreatmentResult} from './treatment' -import {findExperiment, listExperiments} from './experiments' -import {resolveExperimentScenario} from './resolve-experiment-scenario' -import {run} from './run' -import {parseShard, selectShard} from './shard' +import {getEnvironmentConfig} from './environment' +import { + getBenchmark, + run as runBenchmark, + output as getBenchmarkOutput, + serialize as serializeBenchmarkOutput, +} from './benchmark' +import { + getExperiment, + run as runExperiment, + output as getExperimentOutput, + serialize as serializeExperimentOutput, +} from './experiment' +import {logger} from './logger' +import {formatBenchmarkResults, formatExperimentResults} from './report' +import {parseShard} from './shard' +import {compare as compareTrial} from './trial' const {values} = parseArgs({ options: { - artifacts: { + benchmark: { type: 'string', - short: 'a', - description: 'The directory to save artifacts to', + short: 'b', + description: 'The file name of the benchmark to run', + }, + benchmarks: { + type: 'string', + description: 'The directory containing local benchmark files', }, concurrency: { type: 'string', @@ -44,10 +57,17 @@ const {values} = parseArgs({ short: 'h', description: 'Learn more about the command and its options', }, + 'log-level': { + type: 'string', + description: 'The log level to use', + }, output: { type: 'string', description: 'The target file in which results are written', - default: 'output.json', + }, + 'output-dir': { + type: 'string', + description: 'The directory containing output.json and its artifacts', }, scenarios: { type: 'string', @@ -55,465 +75,128 @@ const {values} = parseArgs({ }, shard: { type: 'string', - description: 'The shard to run in / format', + description: 'The experiment shard to run, formatted as order/total', }, }, }) -if (values.help) { +function displayHelp() { console.log(` Usage: agent-eval [options] Options: - -a, --artifacts The directory to save artifacts to + -b, --benchmark The file name of the benchmark to run + --benchmarks The directory containing local benchmark files (default: ./benchmarks) -c, --concurrency The number of treatments to run in parallel --docker-image The Docker container image to use for running treatments (must be a Debian-based Node image with apt-get and a node user, e.g. node:26.5.0-slim) -e, --experiment The file name of the experiment to run - --experiments The directory containing local experiment files + --experiments The directory containing local experiment files (default: ./experiments) -h, --help Learn more about the command and its options + --log-level The log level to use (default: info) --output The target file in which results are written (default: output.json) - --scenarios The directory containing scenario directories - --shard The shard to run + --output-dir The directory containing output.json and its artifacts + --scenarios The directory containing scenario directories (default: ./scenarios) + --shard The experiment shard to run `) +} + +if (values.help) { + displayHelp() process.exit(0) } +if (values['log-level']) { + logger.level = values['log-level'] +} + const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY if (!COPILOT_GITHUB_TOKEN) { - throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run the experiments') + throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval') } -const ARTIFACTS_DIR = path.resolve(values.artifacts ?? 'artifacts') -const DOCKER_IMAGE = values['docker-image']?.trim() || undefined -const parsedConcurrency = values.concurrency ? parseInt(values.concurrency, 10) : 1 -const MAX_CONCURRENCY = - Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1 - ? parsedConcurrency - : 1 -const SHARD = values.shard ? parseShard(values.shard) : undefined -let selectedExperiment: { - id: string - config: ExperimentConfig -} +const env = getEnvironmentConfig({ + benchmarksDirectory: values.benchmarks, + concurrency: values.concurrency, + copilotToken: COPILOT_GITHUB_TOKEN, + dockerImage: values['docker-image']?.trim(), + experimentsDirectory: values.experiments, + outputDirectory: values['output-dir'], + outputPath: values.output, + scenariosDirectory: values.scenarios, +}) -if (!existsSync(ARTIFACTS_DIR)) { - await fs.mkdir(ARTIFACTS_DIR, {recursive: true}) -} +logger.debug('Environment configuration: %o', env) -if (values.experiment) { - const config = await findExperiment(values.experiment, { - directory: values.experiments, - }) - if (!config) { - const experiments = await listExperiments({ - directory: values.experiments, - }) - throw new Error( - `Experiment "${values.experiment}" was not found. Available experiments:\n${experiments - .map(([id]) => id) - .join('\n')}`, - ) - } +if (values.benchmark) { + logger.info('Running benchmark: %s', values.benchmark) - selectedExperiment = { - id: existsSync(values.experiment) - ? path.basename(values.experiment, path.extname(values.experiment)) - : values.experiment, - config, - } -} else { - const experiments = await listExperiments({ - directory: values.experiments, + const benchmark = await getBenchmark({ + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id: values.benchmark, }) - if (experiments.length !== 1) { - throw new Error( - `Select an experiment with --experiment. Available experiments:\n${experiments.map(([id]) => id).join('\n')}`, - ) - } - - selectedExperiment = { - id: experiments[0][0], - config: experiments[0][1], - } -} - -function randomize(input: Array): Array { - const randomized: Array = input.slice() - - // Fisher–Yates shuffle - for (let i = randomized.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)) - ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]] - } - - return randomized -} + const result = await runBenchmark({ + env, + id: values.benchmark, + }) + const sorted = result.toSorted(compareTrial) -function getSuccessRate(result: TreatmentResult): number { - if (result.testResults.numTotalTests === 0) { - return 0 + if (!existsSync(path.dirname(env.outputPath))) { + await fs.mkdir(path.dirname(env.outputPath), {recursive: true}) } - return result.testResults.numPassedTests / result.testResults.numTotalTests -} - -function compareResults(a: TreatmentResult, b: TreatmentResult): number { - return ( - getSuccessRate(b) - getSuccessRate(a) || - a.assistant.outputTokens - b.assistant.outputTokens || - a.assistant.sessionDurationMs - b.assistant.sessionDurationMs || - a.assistant.premiumRequests - b.assistant.premiumRequests || - a.treatment.experiment.name.localeCompare(b.treatment.experiment.name) || - a.treatment.config.name.localeCompare(b.treatment.config.name) || - a.treatment.model.localeCompare(b.treatment.model) || - (a.treatment.reasoningEffort ?? '').localeCompare(b.treatment.reasoningEffort ?? '') || - a.treatment.scenario.id.localeCompare(b.treatment.scenario.id) + logger.info('Writing benchmark output to: %s', env.outputPath) + await fs.writeFile( + env.outputPath, + serializeBenchmarkOutput( + getBenchmarkOutput(benchmark.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }), + ), + 'utf-8', ) -} -type ResultSummary = { - experiment: string - treatment?: string - scenario?: string - model?: Model - reasoningEffort?: ReasoningEffort - runs: number - numPassedTests: number - numTotalTests: number - outputTokens: number - premiumRequests: number - sessionDurationMs: number - totalApiDurationMs: number -} + const resultSummaries = formatBenchmarkResults(benchmark, sorted) + console.log(resultSummaries) -type ResultSummaryValues = { - treatment?: string - scenario?: string - model?: Model - reasoningEffort?: ReasoningEffort -} - -function createResultSummary(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): ResultSummary { - return { - experiment: result.treatment.experiment.name, - treatment: summaryValues.treatment, - scenario: summaryValues.scenario, - model: summaryValues.model, - reasoningEffort: summaryValues.reasoningEffort, - runs: 0, - numPassedTests: 0, - numTotalTests: 0, - outputTokens: 0, - premiumRequests: 0, - sessionDurationMs: 0, - totalApiDurationMs: 0, - } -} - -function addResultToSummary(summary: ResultSummary, result: TreatmentResult) { - summary.runs += 1 - summary.numPassedTests += result.testResults.numPassedTests - summary.numTotalTests += result.testResults.numTotalTests - summary.outputTokens += result.assistant.outputTokens - summary.premiumRequests += result.assistant.premiumRequests - summary.sessionDurationMs += result.assistant.sessionDurationMs - summary.totalApiDurationMs += result.assistant.totalApiDurationMs -} - -function getSummarySuccessRate(summary: ResultSummary): number { - if (summary.numTotalTests === 0) { - return 0 + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } +} else if (values.experiment) { + logger.info('Running experiment: %s', values.experiment) - return summary.numPassedTests / summary.numTotalTests -} - -function compareSummaries(a: ResultSummary, b: ResultSummary): number { - return ( - getSummarySuccessRate(b) - getSummarySuccessRate(a) || - a.outputTokens - b.outputTokens || - a.sessionDurationMs - b.sessionDurationMs || - a.premiumRequests - b.premiumRequests || - a.experiment.localeCompare(b.experiment) || - (a.treatment ?? '').localeCompare(b.treatment ?? '') || - (a.scenario ?? '').localeCompare(b.scenario ?? '') || - (a.model ?? '').localeCompare(b.model ?? '') || - (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '') - ) -} - -function formatPercent(value: number): string { - return `${(value * 100).toFixed(1)}%` -} - -function formatDuration(ms: number): string { - const seconds = ms / 1000 - - if (seconds < 60) { - return `${seconds.toFixed(1)}s` - } - - const minutes = Math.floor(seconds / 60) - const remainingSeconds = seconds - minutes * 60 - return `${minutes}m ${remainingSeconds.toFixed(1)}s` -} - -function formatNumber(value: number): string { - return new Intl.NumberFormat('en-US').format(value) -} - -type TableRow = Record - -function formatTable(rows: Array, columns: Array): string { - const columnWidths = columns.map(column => { - let width = column.length - - for (const row of rows) { - width = Math.max(width, String(row[column] ?? '').length) - } - - return width + const experiment = await getExperiment({ + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id: values.experiment, }) - - const formatRow = (row: TableRow) => { - return columns - .map((column, index) => { - return String(row[column] ?? '').padEnd(columnWidths[index]) - }) - .join(' ') - } - - return [ - formatRow(Object.fromEntries(columns.map(column => [column, column]))), - columnWidths.map(width => '-'.repeat(width)).join(' '), - ...rows.map(formatRow), - ].join('\n') -} - -function getSummaryKey(result: TreatmentResult, summaryValues: ResultSummaryValues = {}): string { - return [ - result.treatment.experiment.name, - summaryValues.treatment ?? '', - summaryValues.scenario ?? '', - summaryValues.model ?? '', - summaryValues.reasoningEffort ?? '', - ].join('\0') -} - -type ResultHierarchy = Array<{ - experiment: string - treatments: Array<{ - summary: ResultSummary - scenarios: Array<{ - summary: ResultSummary - models: Array - }> - }> -}> - -function getResultSummaries(results: Array): ResultHierarchy { - const experiments = new Set() - const treatmentSummaries = new Map() - const scenarioSummaries = new Map() - const modelSummaries = new Map() - - for (const result of results) { - experiments.add(result.treatment.experiment.name) - - const treatmentValues = { - treatment: result.treatment.config.name, - } - const treatmentKey = getSummaryKey(result, treatmentValues) - const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(result, treatmentValues) - addResultToSummary(treatmentSummary, result) - treatmentSummaries.set(treatmentKey, treatmentSummary) - - const scenarioValues = { - treatment: result.treatment.config.name, - scenario: result.treatment.scenario.id, - } - const scenarioKey = getSummaryKey(result, scenarioValues) - const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(result, scenarioValues) - addResultToSummary(scenarioSummary, result) - scenarioSummaries.set(scenarioKey, scenarioSummary) - - const modelValues = { - treatment: result.treatment.config.name, - scenario: result.treatment.scenario.id, - model: result.treatment.model, - reasoningEffort: result.treatment.reasoningEffort, - } - const modelKey = getSummaryKey(result, modelValues) - const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(result, modelValues) - addResultToSummary(modelSummary, result) - modelSummaries.set(modelKey, modelSummary) - } - - return [...experiments].toSorted().map(experiment => { - return { - experiment, - treatments: [...treatmentSummaries.values()] - .filter(treatmentSummary => { - return treatmentSummary.experiment === experiment - }) - .toSorted(compareSummaries) - .map(summary => { - return { - summary, - scenarios: [...scenarioSummaries.values()] - .filter(scenarioSummary => { - return scenarioSummary.experiment === experiment && scenarioSummary.treatment === summary.treatment - }) - .toSorted(compareSummaries) - .map(scenarioSummary => { - return { - summary: scenarioSummary, - models: [...modelSummaries.values()] - .filter(modelSummary => { - return ( - modelSummary.experiment === experiment && - modelSummary.treatment === summary.treatment && - modelSummary.scenario === scenarioSummary.scenario - ) - }) - .toSorted(compareSummaries), - } - }), - } - }), - } + const result = await runExperiment({ + env, + id: values.experiment, + shard: values.shard ? parseShard(values.shard) : undefined, }) -} - -function formatResultSummaries(results: Array): string { - const columns = [ - 'Experiment', - 'Treatment', - 'Scenario', - 'Model', - 'Reasoning Effort', - 'Success Rate', - 'Tests', - 'Runs', - 'Output Tokens', - 'Premium Requests', - 'Session Time', - 'API Time', - ] - const rows: Array = [] + const sorted = result.toSorted(compareTrial) - for (const {treatments} of getResultSummaries(results)) { - for (const {summary, scenarios} of treatments) { - rows.push(formatSummaryRow(summary, 'treatment')) + logger.info('Writing experiment output to: %s', env.outputPath) - for (const {summary: scenarioSummary, models} of scenarios) { - rows.push(formatSummaryRow(scenarioSummary, 'scenario')) + const output = getExperimentOutput(experiment.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }) - for (const model of models) { - rows.push(formatSummaryRow(model, 'model')) - } - } - } + if (!existsSync(path.dirname(env.outputPath))) { + await fs.mkdir(path.dirname(env.outputPath), {recursive: true}) } - return formatTable(rows, columns) -} - -async function appendResultsToJobSummary(resultSummaries: string) { - if (!GITHUB_STEP_SUMMARY) { - return - } + await fs.writeFile(env.outputPath, serializeExperimentOutput(output), 'utf-8') - await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) -} + const resultSummaries = formatExperimentResults(experiment.name, sorted) + console.log(resultSummaries) -function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow { - return { - Experiment: level === 'treatment' ? summary.experiment : '', - Treatment: level === 'treatment' ? (summary.treatment ?? '') : '', - Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? ` ${summary.scenario ?? ''}` : '', - Model: level === 'model' ? ` ${summary.model ?? ''}` : 'All models', - 'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '', - 'Success Rate': formatPercent(getSummarySuccessRate(summary)), - Tests: `${summary.numPassedTests}/${summary.numTotalTests}`, - Runs: summary.runs, - 'Output Tokens': formatNumber(summary.outputTokens), - 'Premium Requests': formatNumber(summary.premiumRequests), - 'Session Time': formatDuration(summary.sessionDurationMs), - 'API Time': formatDuration(summary.totalApiDurationMs), + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } +} else { + displayHelp() } - -const config = selectedExperiment.config - -console.log('Running experiment:', config.name) - -const scenarios = await Promise.all( - config.scenarios.map(scenarioConfig => { - return resolveExperimentScenario(scenarioConfig, { - directory: values.scenarios, - }) - }), -) - -const treatments: Array = config.models.flatMap(modelConfig => { - return resolveModelConfigs(modelConfig).flatMap(({name: model, reasoningEffort}) => { - return scenarios.flatMap(scenarioConfig => { - return [ - { - config: ControlTreatment, - scenario: scenarioConfig, - experiment: config, - id: randomUUID(), - model, - reasoningEffort, - }, - ...config.treatments.map(treatment => { - return { - config: treatment, - scenario: scenarioConfig, - experiment: config, - id: randomUUID(), - model, - reasoningEffort, - } - }), - ] - }) - }) -}) - -// Randomize treatments to mitigate any ordering effects. We want to make sure -// that if there are any external factors that could impact the scenarios (e.g. -// rate limits, resource constraints), they are more likely to impact all -// scenarios rather than just the ones at the end. -const selectedTreatments = SHARD ? selectShard(treatments, SHARD) : treatments -const results: Array = await run(randomize(selectedTreatments), { - artifactsDirectory: ARTIFACTS_DIR, - copilotToken: COPILOT_GITHUB_TOKEN, - dockerImage: DOCKER_IMAGE, - maxConcurrency: MAX_CONCURRENCY, -}) - -const sortedResults = results.toSorted(compareResults) -const resultSummaries = formatResultSummaries(sortedResults) -console.log(resultSummaries) -await appendResultsToJobSummary(resultSummaries) - -const outputFilePath = path.isAbsolute(values.output) ? values.output : path.resolve(process.cwd(), values.output) - -if (!existsSync(path.dirname(outputFilePath))) { - await fs.mkdir(path.dirname(outputFilePath), {recursive: true}) -} - -const output = createAgentEvalOutput({ - id: randomUUID(), - experimentId: selectedExperiment.id, - experiment: config, - scenarios, - results: sortedResults, -}) - -await fs.writeFile(outputFilePath, JSON.stringify(output, null, 2)) diff --git a/packages/agent-eval/src/copilot-cli.test.ts b/packages/agent-eval/src/copilot-cli.test.ts index 4794b432..0c4f1c94 100644 --- a/packages/agent-eval/src/copilot-cli.test.ts +++ b/packages/agent-eval/src/copilot-cli.test.ts @@ -59,6 +59,36 @@ describe(parseMessage, () => { timestamp: '2026-07-24T01:57:27.191Z', parentId: '080d2918-29d2-4da6-b115-3fe37825b5d2', }, + { + type: 'model.message', + data: { + kind: 'message', + turn: 0, + message: { + role: 'assistant', + content: 'Done.', + outputTokens: 42, + }, + }, + ephemeral: true, + id: 'dbf7cd98-649a-44c2-b013-680c902773ac', + timestamp: '2026-09-04T01:06:13.385Z', + parentId: '7e3943af-9aca-4042-b1fd-92a390203597', + }, + { + type: 'user.message', + data: { + content: "Update the index page to use a primary button with the text 'Submit'", + transformedContent: "Update the index page to use a primary button with the text 'Submit'", + supportedNativeDocumentMimeTypes: [], + agentMode: 'autopilot', + interactionId: '40a03f57-636f-41be-bef9-cfe71591862a', + parentAgentTaskId: 'b5c96c9d-a61b-4689-9720-455a5a1e2644', + }, + id: '30d361b9-068e-4ebe-bef0-857a8d92e5ab', + timestamp: '2026-09-01T21:29:34.818Z', + parentId: '0344de20-b203-4642-a644-4cc19a9283ac', + }, { type: 'assistant.turn_start', data: { @@ -168,6 +198,32 @@ describe(parseMessage, () => { timestamp: '2026-08-17T00:23:26.072Z', parentId: '2b1722fe-509e-4368-8c8b-2ae8985099b6', }, + { + type: 'assistant.message', + data: { + messageId: '99e762aa-0196-42be-8756-39ba3c294a07', + content: '', + toolRequests: [ + { + toolCallId: 'call_oLjXYouZFO1PA5D6u8d54cFX', + name: 'glob', + arguments: { + pattern: '**/*', + paths: '/home/sandbox/workspace', + }, + type: 'function', + intentionSummary: '**/*', + }, + ], + interactionId: 'feb477c9-0fb1-47d5-bd08-b397389ec2da', + turnId: '0', + reasoningOpaque: 'opaque', + encryptedContent: 'encrypted', + }, + id: '0c89050a-a6f6-4cd5-8ded-7e25d70b01bc', + timestamp: '2026-09-01T21:35:25.627Z', + parentId: '2ac75cd3-5bf3-4986-a5b2-00cfa4b6e9f5', + }, { type: 'assistant.idle', data: {}, @@ -223,6 +279,29 @@ describe(parseMessage, () => { expect(parseMessage(message)).toEqual(message) }) + test('parses sub-agent user messages without an agent mode', () => { + const message = { + type: 'user.message', + data: { + content: 'Run the build command `npm run build` to see if it succeeds.', + transformedContent: + '2026-09-04T00:43:56.551+00:00\n\nRun the build command `npm run build` to see if it succeeds.', + source: 'agent-f13fa250-5bee-4bfc-8356-5b91a7f72f05', + supportedNativeDocumentMimeTypes: [], + delivery: 'idle', + interactionId: '0c57cce9-3187-4397-b4ae-d6131b882626', + turnId: '0', + parentAgentTaskId: '5bbe4674-f7e7-4ef2-a532-29eaef862529', + }, + agentId: 'f55e8634-e6a7-4b8c-a002-8fa22e9a55cd', + id: 'b8d674d7-f16c-455c-83f5-a7366407445a', + timestamp: '2026-09-04T00:43:56.551Z', + parentId: 'b390bc41-c964-4e13-b3a3-12fb8f2e6303', + } + + expect(parseMessage(message)).toMatchObject(message) + }) + test('does not treat malformed known messages as unrecognized', () => { expect(() => parseMessage({type: 'assistant.turn_start', data: {}})).toThrow() }) diff --git a/packages/agent-eval/src/copilot-cli.ts b/packages/agent-eval/src/copilot-cli.ts index 76982ff9..6f730824 100644 --- a/packages/agent-eval/src/copilot-cli.ts +++ b/packages/agent-eval/src/copilot-cli.ts @@ -46,6 +46,7 @@ const EventFieldsSchema = { id: z.string(), timestamp: z.string(), parentId: z.string(), + agentId: z.optional(z.string()), } const EphemeralEventFieldsSchema = { @@ -95,16 +96,30 @@ const ModelCallStartMessageSchema = z.object({ }), }) +const ModelMessageSchema = z.looseObject({ + type: z.literal('model.message'), + ...EphemeralEventFieldsSchema, + data: z.looseObject({ + message: z.looseObject({ + role: z.string(), + outputTokens: z.optional(z.number()), + }), + }), +}) + const UserMessageSchema = z.object({ type: z.literal('user.message'), ...EventFieldsSchema, data: z.object({ content: z.string(), transformedContent: z.string(), - attachments: z.array(z.unknown()), + attachments: z.optional(z.array(z.unknown())), supportedNativeDocumentMimeTypes: z.array(z.string()), - agentMode: z.string(), + agentMode: z.optional(z.string()), + source: z.optional(z.string()), + delivery: z.optional(z.string()), interactionId: z.string(), + turnId: z.optional(z.string()), parentAgentTaskId: z.string(), }), }) @@ -149,7 +164,7 @@ const AssistantMessageSchema = z.object({ encryptedContent: z.optional(z.string()), phase: z.optional(z.string()), outputTokens: z.optional(z.number()), - requestId: z.string(), + requestId: z.optional(z.string()), }), }) @@ -298,12 +313,15 @@ const ResultMessageSchema = z.object({ }), }) +type ResultMessage = z.infer + const KnownMessageSchema = z.discriminatedUnion('type', [ SessionMcpServerStatusChangedMessageSchema, SessionMcpServersLoadedMessageSchema, SessionSkillsLoadedMessageSchema, SessionToolsUpdatedMessageSchema, ModelCallStartMessageSchema, + ModelMessageSchema, UserMessageSchema, AssistantTurnStartMessageSchema, AssistantMessageStartMessageSchema, @@ -330,6 +348,7 @@ const KNOWN_MESSAGE_TYPES = new Set([ 'session.skills_loaded', 'session.tools_updated', 'model.call_start', + 'model.message', 'user.message', 'assistant.turn_start', 'assistant.message_start', @@ -390,4 +409,4 @@ export { isMessageType, parseMessage, } -export type {KnownMessage, Message, UnknownMessage, UnknownMessageType} +export type {KnownMessage, Message, ResultMessage, UnknownMessage, UnknownMessageType} diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts new file mode 100644 index 00000000..b6ded377 --- /dev/null +++ b/packages/agent-eval/src/environment.test.ts @@ -0,0 +1,83 @@ +import path from 'node:path' +import {describe, expect, test} from 'vitest' +import {DEFAULT_DOCKER_IMAGE} from './sandbox' +import {getEnvironmentConfig} from './environment' + +describe('getEnvironmentConfig', () => { + test('resolves default paths and values', () => { + expect( + getEnvironmentConfig({ + copilotToken: 'token', + }), + ).toEqual({ + artifactsDirectory: path.resolve('artifacts'), + benchmarksDirectory: path.resolve('benchmarks'), + concurrency: 1, + copilotToken: 'token', + dockerImage: DEFAULT_DOCKER_IMAGE, + experimentsDirectory: path.resolve('experiments'), + outputPath: path.resolve('output.json'), + scenariosDirectory: path.resolve('scenarios'), + }) + }) + + test('uses valid custom values', () => { + expect( + getEnvironmentConfig({ + benchmarksDirectory: './custom-benchmarks', + concurrency: '4', + copilotToken: 'token', + dockerImage: 'node:custom', + experimentsDirectory: './custom-experiments', + outputPath: './results/output.json', + scenariosDirectory: './custom-scenarios', + }), + ).toEqual({ + artifactsDirectory: path.resolve('results/artifacts'), + benchmarksDirectory: path.resolve('custom-benchmarks'), + concurrency: 4, + copilotToken: 'token', + dockerImage: 'node:custom', + experimentsDirectory: path.resolve('custom-experiments'), + outputPath: path.resolve('results/output.json'), + scenariosDirectory: path.resolve('custom-scenarios'), + }) + }) + + test('derives the output and artifacts paths from an output directory', () => { + expect( + getEnvironmentConfig({ + copilotToken: 'token', + outputDirectory: './results/run', + }), + ).toEqual({ + artifactsDirectory: path.resolve('results/run/artifacts'), + benchmarksDirectory: path.resolve('benchmarks'), + concurrency: 1, + copilotToken: 'token', + dockerImage: DEFAULT_DOCKER_IMAGE, + experimentsDirectory: path.resolve('experiments'), + outputPath: path.resolve('results/run/output.json'), + scenariosDirectory: path.resolve('scenarios'), + }) + }) + + test('rejects output directory combinations with explicit output paths', () => { + expect(() => { + getEnvironmentConfig({ + copilotToken: 'token', + outputDirectory: './results/run', + outputPath: './output.json', + }) + }).toThrow('--output-dir cannot be combined with --output') + }) + + test.each(['0', '-1', 'invalid', '1.5'])('falls back to one for invalid concurrency %s', concurrency => { + expect( + getEnvironmentConfig({ + concurrency, + copilotToken: 'token', + }).concurrency, + ).toBe(1) + }) +}) diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts new file mode 100644 index 00000000..8ce00d19 --- /dev/null +++ b/packages/agent-eval/src/environment.ts @@ -0,0 +1,58 @@ +import path from 'node:path' +import {DEFAULT_DOCKER_IMAGE} from './sandbox' + +type EnvironmentConfig = { + artifactsDirectory: string + benchmarksDirectory: string + concurrency: number + copilotToken: string + dockerImage: string + experimentsDirectory: string + outputPath: string + scenariosDirectory: string +} + +type EnvironmentOptions = { + benchmarksDirectory?: string + concurrency?: string + copilotToken: string + dockerImage?: string + experimentsDirectory?: string + outputDirectory?: string + outputPath?: string + scenariosDirectory?: string +} + +function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig { + if (options.outputDirectory && options.outputPath) { + throw new Error('--output-dir cannot be combined with --output') + } + + const outputDirectory = options.outputDirectory ? path.resolve(options.outputDirectory) : undefined + const benchmarksDirectory = path.resolve(options.benchmarksDirectory ?? 'benchmarks') + const parsedConcurrency = options.concurrency ? parseInt(options.concurrency, 10) : 1 + const concurrency = + Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1 + ? parsedConcurrency + : 1 + const experimentsDirectory = path.resolve(options.experimentsDirectory ?? 'experiments') + const outputPath = outputDirectory + ? path.join(outputDirectory, 'output.json') + : path.resolve(options.outputPath ?? 'output.json') + const artifactsDirectory = path.join(path.dirname(outputPath), 'artifacts') + const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios') + + return { + artifactsDirectory, + benchmarksDirectory, + concurrency, + copilotToken: options.copilotToken, + dockerImage: options.dockerImage ?? DEFAULT_DOCKER_IMAGE, + experimentsDirectory, + outputPath, + scenariosDirectory, + } +} + +export {getEnvironmentConfig} +export type {EnvironmentConfig} diff --git a/packages/agent-eval/src/experiment-config.ts b/packages/agent-eval/src/experiment-config.ts deleted file mode 100644 index 6ccbea10..00000000 --- a/packages/agent-eval/src/experiment-config.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type {ExperimentModelConfig} from './model' -import type {Sandbox} from './sandbox' - -type ScenarioConfig = { - description?: string - prompt: string - tags?: Array -} - -type InlineScenarioConfig = { - name?: string - path: string -} - -type ExperimentScenarioConfig = string | InlineScenarioConfig - -type ExperimentConfig = { - name: string - description: string - models: Array - scenarios: Array - setup?: Setup - treatments: Array -} - -type TreatmentConfig = { - name: string - setup?: Setup -} - -type Setup = ({sandbox}: {sandbox: Sandbox}) => Promise - -const ControlTreatment: TreatmentConfig = { - name: 'Control', -} - -export {ControlTreatment} -export type {ExperimentConfig, ExperimentScenarioConfig, InlineScenarioConfig, ScenarioConfig, TreatmentConfig} diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts new file mode 100644 index 00000000..3b5fad20 --- /dev/null +++ b/packages/agent-eval/src/experiment.test.ts @@ -0,0 +1,371 @@ +import path from 'node:path' +import {expect, test} from 'vitest' +import {defineConfig, deserialize, getExperiment, listExperiments, output, serialize} from './experiment' +import {VirtualHost} from './host' +import type {TrialResult} from './trial' + +const config = defineConfig({ + name: 'Test experiment', + description: 'Tests an experiment', + models: ['gpt-5.6-sol'], + scenarios: [], + treatments: [ + { + name: 'Test treatment', + }, + ], +}) + +function resolveConfig(input: typeof config) { + return { + ...input, + models: [ + { + name: 'gpt-5.6-sol' as const, + reasoningEffort: 'medium' as const, + }, + ], + setup: undefined, + } +} + +test('listExperiments loads named and default exports from supported files', async () => { + const serializedConfig = JSON.stringify(config) + const host = VirtualHost.create({ + '/experiments': { + 'named.ts': `export const experiment = ${serializedConfig}`, + 'default.js': `export default ${serializedConfig}`, + 'commonjs.cjs': `export default ${serializedConfig}`, + 'module.mjs': `export default ${serializedConfig}`, + }, + }) + + const experiments = await listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }) + + expect(experiments).toHaveLength(4) + expect(experiments).toEqual( + expect.arrayContaining([ + { + ...resolveConfig(config), + id: 'named', + filepath: '/experiments/named.ts', + }, + { + ...resolveConfig(config), + id: 'default', + filepath: '/experiments/default.js', + }, + { + ...resolveConfig(config), + id: 'commonjs', + filepath: '/experiments/commonjs.cjs', + }, + { + ...resolveConfig(config), + id: 'module', + filepath: '/experiments/module.mjs', + }, + ]), + ) +}) + +test('listExperiments prefers the named experiment export', async () => { + const namedConfig = { + ...config, + name: 'Named experiment', + } + const defaultConfig = { + ...config, + name: 'Default experiment', + } + const host = VirtualHost.create({ + '/experiments': { + 'experiment.ts': [ + `export const experiment = ${JSON.stringify(namedConfig)}`, + `export default ${JSON.stringify(defaultConfig)}`, + ].join('\n'), + }, + }) + + await expect( + listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }), + ).resolves.toEqual([ + { + ...resolveConfig(namedConfig), + id: 'experiment', + filepath: '/experiments/experiment.ts', + }, + ]) +}) + +test('listExperiments sorts experiments by filename', async () => { + const serializedConfig = JSON.stringify(config) + const host = VirtualHost.create({ + '/experiments': { + 'z-last.ts': `export const experiment = ${serializedConfig}`, + 'a-first.ts': `export const experiment = ${serializedConfig}`, + }, + }) + + const experiments = await listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }) + + expect( + experiments.map(experiment => { + return experiment.id + }), + ).toEqual(['a-first', 'z-last']) +}) + +test('listExperiments resolves inline scenario paths with optional names', async () => { + const unnamedDirectory = path.resolve('/fixtures/unnamed-scenario') + const namedDirectory = path.resolve('/fixtures/named-scenario') + const inlineConfig = { + ...config, + scenarios: [ + { + path: unnamedDirectory, + }, + { + name: 'custom-name', + path: namedDirectory, + }, + ], + } + const scenarioConfig = JSON.stringify({ + prompt: 'Complete the task', + }) + const host = VirtualHost.create({ + '/experiments': { + 'inline.ts': `export default ${JSON.stringify(inlineConfig)}`, + }, + '/fixtures': { + 'unnamed-scenario': { + 'scenario.browser.test.ts': '', + 'scenario.config.ts': `export default ${scenarioConfig}`, + 'scenario.test.ts': '', + }, + 'named-scenario': { + 'scenario.config.ts': `export default ${scenarioConfig}`, + 'scenario.test.ts': '', + }, + }, + }) + + await expect( + listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }), + ).resolves.toEqual([ + { + ...resolveConfig(inlineConfig), + id: 'inline', + filepath: '/experiments/inline.ts', + scenarios: [ + { + id: 'unnamed-scenario', + directory: unnamedDirectory, + prompt: 'Complete the task', + tags: [], + testPath: path.join(unnamedDirectory, 'scenario.test.ts'), + browserTestPath: path.join(unnamedDirectory, 'scenario.browser.test.ts'), + }, + { + id: 'custom-name', + directory: namedDirectory, + prompt: 'Complete the task', + tags: [], + testPath: path.join(namedDirectory, 'scenario.test.ts'), + }, + ], + }, + ]) +}) + +test('listExperiments ignores unsupported, reserved, missing, and invalid configs', async () => { + const serializedConfig = JSON.stringify(config) + const host = VirtualHost.create({ + '/experiments': { + 'valid.ts': `export const experiment = ${serializedConfig}`, + 'types.d.ts': `export const experiment = ${serializedConfig}`, + 'index.ts': `export const experiment = ${serializedConfig}`, + 'unsupported.json': serializedConfig, + 'missing.ts': 'export const value = true', + 'invalid.ts': 'export const experiment = {}', + }, + }) + + await expect( + listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }), + ).resolves.toEqual([ + { + ...resolveConfig(config), + id: 'valid', + filepath: '/experiments/valid.ts', + }, + ]) +}) + +test('listExperiments throws when the directory does not exist', async () => { + const host = VirtualHost.create() + + await expect( + listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }), + ).rejects.toThrow('Experiments directory does not exist: /experiments') +}) + +test('listExperiments throws when the path is not a directory', async () => { + const host = VirtualHost.create({ + '/experiments': '', + }) + + await expect( + listExperiments({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + }), + ).rejects.toThrow('Experiments path is not a directory: /experiments') +}) + +test('getExperiment returns the experiment matching the id', async () => { + const serializedConfig = JSON.stringify(config) + const host = VirtualHost.create({ + '/experiments': { + 'first.ts': `export const experiment = ${serializedConfig}`, + 'second.ts': `export const experiment = ${serializedConfig}`, + }, + }) + + await expect( + getExperiment({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + id: 'second', + }), + ).resolves.toEqual({ + ...resolveConfig(config), + id: 'second', + filepath: '/experiments/second.ts', + }) +}) + +test('getExperiment throws when the experiment is not found', async () => { + const host = VirtualHost.create({ + '/experiments': {}, + }) + + await expect( + getExperiment({ + host, + experimentsDirectory: '/experiments', + scenariosDirectory: '/scenarios', + id: 'missing', + }), + ).rejects.toThrow('Experiment "missing" was not found in: /experiments') +}) + +test('serializes and deserializes experiment identity and result maps', () => { + const experimentOutput = output('baseline', []) + const serialized = serialize(experimentOutput) + + expect(JSON.parse(serialized)).toEqual({ + experimentId: 'baseline', + scenarios: {}, + treatments: {}, + trials: {}, + }) + expect(deserialize(serialized)).toEqual({ + experimentId: 'baseline', + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + }) +}) + +test('creates portable artifact paths relative to the output directory', () => { + const trialResult: TrialResult = { + artifacts: { + directory: '/bundle/artifacts/trial', + copilotConfigDirectory: '/bundle/artifacts/trial/.copilot', + skillsConfigDirectory: '/bundle/artifacts/trial/.agents', + testResultsPath: '/bundle/artifacts/trial/workspace/test-results.json', + workspaceDirectory: '/bundle/artifacts/trial/workspace', + }, + trial: { + id: 'trial', + scenario: { + id: 'scenario', + directory: '/scenarios/scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/scenario/scenario.test.ts', + }, + treatment: { + name: 'Control', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + }, + agent: { + sessions: [], + }, + testResults: { + numTotalTests: 1, + numPassedTests: 1, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }, + walkthrough: { + type: 'Screenshot', + filepath: '/bundle/artifacts/trial/walkthrough/screenshot.png', + }, + } + + const portableOutput = output('baseline', [trialResult], { + baseDirectory: '/bundle', + }) + + expect(portableOutput.trials.get('trial')).toEqual( + expect.objectContaining({ + artifacts: { + directory: 'artifacts/trial', + copilotConfigDirectory: 'artifacts/trial/.copilot', + skillsConfigDirectory: 'artifacts/trial/.agents', + testResultsPath: 'artifacts/trial/workspace/test-results.json', + workspaceDirectory: 'artifacts/trial/workspace', + }, + walkthrough: { + type: 'Screenshot', + filepath: 'artifacts/trial/walkthrough/screenshot.png', + }, + }), + ) +}) diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts index 5cbf7f20..1cd3875a 100644 --- a/packages/agent-eval/src/experiment.ts +++ b/packages/agent-eval/src/experiment.ts @@ -1,31 +1,341 @@ -import type {ExperimentConfig, TreatmentConfig} from './experiment-config' -import type { - CopilotPluginConfig, - CopilotPluginSource, - LocalCopilotPluginSource, - McpServerConfig, - RemoteCopilotPluginSource, - Sandbox, -} from './sandbox' -import type {ExperimentModelConfig, Model, ModelConfig, ModelInfo, ReasoningEffort} from './model' +import {randomUUID} from 'node:crypto' +import path from 'node:path' +import * as z from 'zod/mini' +import type {EnvironmentConfig} from './environment' +import { + getModelVariants, + ModelVariantConfigSchema, + ModelVariantSchema, + type ModelVariant, + type ModelVariantConfig, +} from './model' +import {DefaultHost, type Host} from './host' +import {logger} from './logger' +import {create as createPlan, run as runPlan} from './plan' +import {getScenario, loadScenario, ScenarioSchema, type Scenario} from './scenario' +import {selectShard, type Shard} from './shard' +import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' +import { + getPortableTrialPaths, + TrialAgentSchema, + TrialArtifactsSchema, + WalkthroughSchema, + type Trial, + type TrialResult, +} from './trial' +import {TestResultsSchema} from './vitest' + +type InlineScenarioConfig = { + name?: string + path: string +} + +type ExperimentScenarioConfig = string | InlineScenarioConfig + +type ExperimentConfig = { + name: string + description: string + models: ModelVariantConfig + scenarios: Array + setup?: TreatmentSetup + treatments: Array +} + +const InlineScenarioConfigSchema = z.object({ + name: z.optional(z.string()), + path: z.string(), +}) + +const ExperimentConfigSchema = z.object({ + name: z.string(), + description: z.string(), + models: ModelVariantConfigSchema, + scenarios: z.array(z.union([z.string(), InlineScenarioConfigSchema])), + setup: z.optional(TreatmentSetupSchema), + treatments: z.array(TreatmentSchema), +}) satisfies z.ZodMiniType function defineConfig(config: ExperimentConfig): ExperimentConfig { return config } +type Experiment = { + id: string + filepath: string + name: ExperimentConfig['name'] + description: ExperimentConfig['description'] + models: Array + scenarios: Array + setup?: TreatmentSetup + treatments: Array +} + +type ExperimentModule = { + default?: unknown + experiment?: unknown +} + +const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts']) + +async function listExperiments({ + host = DefaultHost, + experimentsDirectory, + scenariosDirectory, +}: { + host?: Host + experimentsDirectory: string + scenariosDirectory: string +}): Promise> { + if (!host.existsSync(experimentsDirectory)) { + throw new Error(`Experiments directory does not exist: ${experimentsDirectory}`) + } + + const stats = await host.fs.stat(experimentsDirectory) + if (!stats.isDirectory()) { + throw new Error(`Experiments path is not a directory: ${experimentsDirectory}`) + } + + const filenames = (await host.fs.readdir(experimentsDirectory)).sort() + const experiments: Array = [] + + for (const filename of filenames) { + if ( + filename.endsWith('.d.ts') || + filename === 'index.ts' || + !EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename)) + ) { + continue + } + + const filepath = path.join(experimentsDirectory, filename) + const mod: ExperimentModule = await host.loadModule(filepath) + const data = mod.experiment ?? mod.default + if (!data) { + continue + } + + const parseResult = ExperimentConfigSchema.safeParse(data) + if (!parseResult.success) { + logger.warn( + `Failed to parse experiment config for file: ${filepath}. Error: ${z.prettifyError(parseResult.error)}`, + ) + continue + } + + const {data: config} = parseResult + const scenarios = await Promise.all( + config.scenarios.map(scenario => { + if (typeof scenario === 'string') { + return getScenario(host, scenariosDirectory, scenario) + } + + const directory = path.resolve(scenario.path) + return loadScenario(host, directory, scenario.name ?? path.basename(directory)) + }), + ) + const id = path.basename(filename, path.extname(filename)) + const experiment: Experiment = { + id, + filepath, + name: config.name, + description: config.description, + models: getModelVariants(config.models), + scenarios, + setup: config.setup, + treatments: config.treatments, + } + experiments.push(experiment) + } + + return experiments +} + +async function getExperiment({ + host = DefaultHost, + experimentsDirectory, + scenariosDirectory, + id, +}: { + host?: Host + experimentsDirectory: string + scenariosDirectory: string + id: string +}): Promise { + const experiments = await listExperiments({ + host, + experimentsDirectory, + scenariosDirectory, + }) + const experiment = experiments.find(candidate => candidate.id === id) + if (experiment) { + return experiment + } + + throw new Error(`Experiment "${id}" was not found in: ${experimentsDirectory}`) +} + +type ExperimentRunResult = Array + +async function run({ + env, + host = DefaultHost, + id, + shard, +}: { + env: EnvironmentConfig + host?: Host + id: string + shard?: Shard +}): Promise { + const experiment = await getExperiment({ + host, + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id, + }) + const trials: Array = experiment.models.flatMap(model => { + return experiment.scenarios.flatMap(scenario => { + return [ + { + id: randomUUID(), + scenario, + treatment: ControlTreatment, + model, + setup: experiment.setup, + }, + ...experiment.treatments.map(treatment => { + return { + id: randomUUID(), + scenario, + treatment, + model, + setup: experiment.setup, + } + }), + ] + }) + }) + const plan = await createPlan(shard ? selectShard(trials, shard) : trials) + const results = await runPlan({ + env, + host, + plan, + }) + + return results +} + +const ExperimentOutputScenarioSchema = z.pick(ScenarioSchema, { + id: true, + directory: true, + prompt: true, + description: true, + tags: true, + testPath: true, + browserTestPath: true, +}) + +const ExperimentOutputTreatmentSchema = z.pick(TreatmentSchema, { + name: true, +}) + +const ExperimentOutputTrialSchema = z.object({ + agent: TrialAgentSchema, + artifacts: TrialArtifactsSchema, + id: z.string(), + model: ModelVariantSchema, + scenarioId: z.string(), + testResults: TestResultsSchema, + treatmentId: z.string(), + walkthrough: WalkthroughSchema, +}) + +type ExperimentOutput = { + experimentId: string + scenarios: Map> + treatments: Map> + trials: Map> +} + +type ExperimentOutputOptions = { + baseDirectory?: string +} + +const SerializedExperimentOutputSchema = z.object({ + experimentId: z.string(), + scenarios: z.record(z.string(), ExperimentOutputScenarioSchema), + treatments: z.record(z.string(), ExperimentOutputTreatmentSchema), + trials: z.record(z.string(), ExperimentOutputTrialSchema), +}) + +function output( + experimentId: string, + trialResults: ExperimentRunResult, + options: ExperimentOutputOptions = {}, +): ExperimentOutput { + const result: ExperimentOutput = { + experimentId, + scenarios: new Map(), + treatments: new Map(), + trials: new Map(), + } + + for (const trialResult of trialResults) { + const {trial} = trialResult + const {artifacts, walkthrough} = options.baseDirectory + ? getPortableTrialPaths(trialResult, options.baseDirectory) + : trialResult + + if (!result.scenarios.has(trial.scenario.id)) { + result.scenarios.set(trial.scenario.id, trial.scenario) + } + + if (!result.treatments.has(trial.treatment.name)) { + result.treatments.set(trial.treatment.name, trial.treatment) + } + + result.trials.set(trial.id, { + agent: trialResult.agent, + artifacts, + id: trial.id, + model: trial.model, + scenarioId: trial.scenario.id, + testResults: trialResult.testResults, + treatmentId: trial.treatment.name, + walkthrough, + }) + } + + return result +} + +function serialize(experimentOutput: ExperimentOutput): string { + return JSON.stringify({ + experimentId: experimentOutput.experimentId, + scenarios: Object.fromEntries(experimentOutput.scenarios), + treatments: Object.fromEntries(experimentOutput.treatments), + trials: Object.fromEntries(experimentOutput.trials), + }) +} + +function deserialize(input: unknown): ExperimentOutput { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + const result = SerializedExperimentOutputSchema.parse(parsed, {reportInput: true}) + + return { + experimentId: result.experimentId, + scenarios: new Map(Object.entries(result.scenarios)), + treatments: new Map(Object.entries(result.treatments)), + trials: new Map(Object.entries(result.trials)), + } +} + +export {ExperimentConfigSchema, defineConfig, deserialize, getExperiment, listExperiments, output, run, serialize} export type { ExperimentConfig, - ExperimentModelConfig, - CopilotPluginConfig, - CopilotPluginSource, - LocalCopilotPluginSource, - McpServerConfig, - Model, - ModelConfig, - ModelInfo, - ReasoningEffort, - RemoteCopilotPluginSource, - Sandbox, - TreatmentConfig, -} -export {defineConfig} + Experiment, + ExperimentOutput, + ExperimentOutputOptions, + ExperimentScenarioConfig, + InlineScenarioConfig, +} diff --git a/packages/agent-eval/src/experiments.test.ts b/packages/agent-eval/src/experiments.test.ts deleted file mode 100644 index 005ed2fd..00000000 --- a/packages/agent-eval/src/experiments.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import fs from 'node:fs/promises' -import os from 'node:os' -import path from 'node:path' -import {describe, expect, test} from 'vitest' -import {findExperiment, listExperiments, loadExperimentConfigs} from './experiments' - -async function createExperimentsDirectory() { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-experiments-')) - await fs.writeFile( - path.join(directory, 'example.mjs'), - `export const experiment = { - name: 'Example', - description: 'Example experiment', - models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], - scenarios: ['001-agent-uses-button-from-primer'], - treatments: [] - }`, - ) - await fs.writeFile( - path.join(directory, 'default-export.mjs'), - `export default { - name: 'Default export', - description: 'Default export experiment', - models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], - scenarios: ['001-agent-uses-button-from-primer'], - treatments: [] - }`, - ) - await fs.writeFile(path.join(directory, 'index.ts'), 'throw new Error("index should be ignored")') - return directory -} - -describe('local experiment loading', () => { - test('lists experiments from a local directory', async () => { - const directory = await createExperimentsDirectory() - - await expect(listExperiments({directory})).resolves.toEqual([ - ['default-export', expect.objectContaining({name: 'Default export'})], - ['example', expect.objectContaining({name: 'Example'})], - ]) - }) - - test('finds a named experiment from a local directory', async () => { - const directory = await createExperimentsDirectory() - - await expect(findExperiment('example', {directory})).resolves.toEqual(expect.objectContaining({name: 'Example'})) - }) - - test('finds an experiment from a local file path', async () => { - const directory = await createExperimentsDirectory() - - await expect(findExperiment(path.join(directory, 'example.mjs'))).resolves.toEqual( - expect.objectContaining({name: 'Example'}), - ) - }) - - test('returns undefined when an experiment is not found', async () => { - const directory = await createExperimentsDirectory() - - await expect(findExperiment('missing', {directory})).resolves.toBeUndefined() - }) - - test('loads an experiment from a local file path', async () => { - const directory = await createExperimentsDirectory() - - await expect(loadExperimentConfigs({experiment: path.join(directory, 'example.mjs')})).resolves.toEqual([ - expect.objectContaining({name: 'Example'}), - ]) - }) - - test('loads all experiments when no experiment is specified', async () => { - const directory = await createExperimentsDirectory() - - await expect(loadExperimentConfigs({directory})).resolves.toHaveLength(2) - }) -}) diff --git a/packages/agent-eval/src/experiments.ts b/packages/agent-eval/src/experiments.ts deleted file mode 100644 index 581f7f1f..00000000 --- a/packages/agent-eval/src/experiments.ts +++ /dev/null @@ -1,91 +0,0 @@ -import {existsSync} from 'node:fs' -import fs from 'node:fs/promises' -import path from 'node:path' -import {pathToFileURL} from 'node:url' -import type {ExperimentConfig} from './experiment-config' - -type ExperimentModule = { - default?: ExperimentConfig - experiment?: ExperimentConfig -} - -type ExperimentSourceOptions = { - directory?: string -} - -type LoadExperimentOptions = ExperimentSourceOptions & { - experiment?: string -} - -const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts']) - -function getExperimentId(filename: string): string { - return filename.replace(/\.(?:cjs|js|mjs|ts)$/, '') -} - -function isExperimentFile(filename: string): boolean { - if (filename.endsWith('.d.ts')) { - return false - } - - return filename !== 'index.ts' && EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename)) -} - -async function loadExperimentFile(filepath: string): Promise { - const resolvedPath = path.resolve(filepath) - const mod = (await import(pathToFileURL(resolvedPath).href)) as ExperimentModule - const experiment = mod.experiment ?? mod.default - if (!experiment) { - throw new Error(`Experiment file must export "experiment" or a default export: ${resolvedPath}`) - } - - return experiment -} - -async function getExperimentEntries(sourceDirectory: string): Promise> { - const directory = path.resolve(sourceDirectory) - if (!existsSync(directory)) { - throw new Error(`Experiments directory does not exist: ${directory}`) - } - - const stats = await fs.stat(directory) - if (!stats.isDirectory()) { - throw new Error(`Experiments path is not a directory: ${directory}`) - } - - const filenames = (await fs.readdir(directory)).toSorted() - return Promise.all( - filenames.filter(isExperimentFile).map(async filename => { - const filepath = path.join(directory, filename) - return [getExperimentId(filename), await loadExperimentFile(filepath)] - }), - ) -} - -async function listExperiments(options: ExperimentSourceOptions = {}): Promise> { - return getExperimentEntries(options.directory ?? 'experiments') -} - -async function findExperiment( - id: string, - options: ExperimentSourceOptions = {}, -): Promise { - if (existsSync(id)) { - return loadExperimentFile(id) - } - - const experiments = await getExperimentEntries(options.directory ?? 'experiments') - return experiments.find(([name]) => name === id)?.[1] -} - -async function loadExperimentConfigs(options: LoadExperimentOptions = {}): Promise> { - if (options.experiment) { - const experiment = await findExperiment(options.experiment, options) - return experiment ? [experiment] : [] - } - - return (await listExperiments(options)).map(([, experiment]) => experiment) -} - -export {findExperiment, listExperiments, loadExperimentConfigs} -export type {ExperimentSourceOptions, LoadExperimentOptions} diff --git a/packages/agent-eval/src/host.test.ts b/packages/agent-eval/src/host.test.ts new file mode 100644 index 00000000..8ba72da0 --- /dev/null +++ b/packages/agent-eval/src/host.test.ts @@ -0,0 +1,56 @@ +import os from 'node:os' +import path from 'node:path' +import fs from 'node:fs/promises' +import {pathToFileURL} from 'node:url' +import {expect, test} from 'vitest' +import {DefaultHost, SystemHost, VirtualHost} from './host' + +test('SystemHost provides access to the system filesystem and module loader', async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-host-')) + const textFilepath = path.join(directory, 'example.txt') + const moduleFilepath = path.join(directory, 'example.mjs') + + try { + await fs.writeFile(textFilepath, 'example') + await fs.writeFile(moduleFilepath, 'export const value = "loaded"') + + const host = await SystemHost.create() + + expect(host.existsSync(textFilepath)).toBe(true) + await expect(host.fs.readFile(textFilepath, 'utf8')).resolves.toBe('example') + await expect(host.loadModule<{value: string}>(moduleFilepath)).resolves.toMatchObject({ + value: 'loaded', + }) + await expect(host.loadModule<{value: string}>(pathToFileURL(moduleFilepath).href)).resolves.toMatchObject({ + value: 'loaded', + }) + } finally { + await fs.rm(directory, {recursive: true, force: true}) + } +}) + +test('DefaultHost is a system host', () => { + expect(DefaultHost).toBeInstanceOf(SystemHost) +}) + +test('VirtualHost provides access to an in-memory filesystem and module loader', async () => { + const host = VirtualHost.create({ + '/example.txt': 'example', + '/example.mjs': 'export const value = "loaded"', + }) + + expect(host.existsSync('/example.txt')).toBe(true) + expect(host.existsSync('/missing.txt')).toBe(false) + await expect(host.fs.readFile('/example.txt', 'utf8')).resolves.toBe('example') + await expect(host.loadModule<{value: string}>('/example.mjs')).resolves.toMatchObject({ + value: 'loaded', + }) +}) + +test('VirtualHost creates an empty writable filesystem by default', async () => { + const host = VirtualHost.create() + + await host.fs.writeFile('/example.txt', 'example') + + expect(host.vol.readFileSync('/example.txt', 'utf8')).toBe('example') +}) diff --git a/packages/agent-eval/src/host.ts b/packages/agent-eval/src/host.ts new file mode 100644 index 00000000..cbf23747 --- /dev/null +++ b/packages/agent-eval/src/host.ts @@ -0,0 +1,76 @@ +import {existsSync} from 'node:fs' +import fs from 'node:fs/promises' +import {pathToFileURL} from 'node:url' +import {memfs, Volume, type NestedDirectoryJSON} from 'memfs' +import {SystemSandbox, VirtualSandbox, type Sandbox, type SandboxCreateOptions} from './sandbox' + +type FileSystem = typeof import('node:fs/promises') + +interface Host { + existsSync: typeof existsSync + fs: FileSystem + loadModule(filepath: string): Promise + createSandbox: (options?: SandboxCreateOptions) => Promise +} + +class SystemHost implements Host { + static async create() { + return new SystemHost() + } + + existsSync: typeof existsSync + fs: FileSystem + + constructor() { + this.existsSync = existsSync + this.fs = fs + } + + loadModule(filepath: string): Promise { + const specifier = + filepath.startsWith('file:') || filepath.startsWith('data:') ? filepath : pathToFileURL(filepath).href + return import(specifier) + } + + createSandbox(options?: SandboxCreateOptions): Promise { + return SystemSandbox.create(options) + } +} + +class VirtualHost implements Host { + static create(json?: NestedDirectoryJSON) { + return new VirtualHost(json) + } + + existsSync: typeof existsSync + fs: FileSystem + vol: Volume + + constructor(json?: NestedDirectoryJSON) { + const {fs: virtualFs, vol} = memfs(json) + this.vol = vol + this.existsSync = virtualFs.existsSync + // @ts-expect-error - not every constant is exposed but memfs should match + // for our test cases + this.fs = virtualFs.promises + } + + async loadModule(filepath: string): Promise { + const contents = await this.fs.readFile(filepath, 'utf-8') + const encodedContent = Buffer.from(contents).toString('base64') + const dataUri = `data:text/javascript;base64,${encodedContent}` + return await import(dataUri) + } + + createSandbox(options?: SandboxCreateOptions): Promise { + return VirtualSandbox.create({ + host: this, + ...options, + }) + } +} + +const DefaultHost = new SystemHost() + +export {SystemHost, VirtualHost, DefaultHost} +export type {Host} diff --git a/packages/agent-eval/src/index.test.ts b/packages/agent-eval/src/index.test.ts new file mode 100644 index 00000000..c74c9077 --- /dev/null +++ b/packages/agent-eval/src/index.test.ts @@ -0,0 +1,25 @@ +import {expect, test} from 'vitest' +import { + BenchmarkConfigSchema, + ControlTreatment, + ExperimentConfigSchema, + ScenarioConfigSchema, + TreatmentSchema, + TrialResultSchema, + TrialSchema, + defineBenchmarkConfig, + defineExperimentConfig, + defineScenarioConfig, +} from './index' + +test('exports the public configuration helpers and schemas', () => { + expect(defineBenchmarkConfig).toBeTypeOf('function') + expect(defineExperimentConfig).toBeTypeOf('function') + expect(defineScenarioConfig).toBeTypeOf('function') + expect(BenchmarkConfigSchema).toBeDefined() + expect(ExperimentConfigSchema).toBeDefined() + expect(ScenarioConfigSchema).toBeDefined() + expect(TreatmentSchema.parse(ControlTreatment)).toEqual(ControlTreatment) + expect(TrialSchema).toBeDefined() + expect(TrialResultSchema).toBeDefined() +}) diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index 8e0a2f11..39dd3dc2 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -1,19 +1,39 @@ -export {findExperiment, listExperiments, loadExperimentConfigs} from './experiments' -export type {ExperimentSourceOptions, LoadExperimentOptions} from './experiments' -export {findScenario, listScenarios} from './scenarios' -export type {ResolvedScenario, ScenarioSourceOptions} from './scenarios' -export {run} from './run' -export type {Treatment, TreatmentResult} from './treatment' -export {defineConfig} from './experiment' -export {models} from './model' +export { + BenchmarkConfigSchema, + defineConfig as defineBenchmarkConfig, + deserialize as deserializeBenchmarkOutput, + getBenchmark, + listBenchmarks, + output as getBenchmarkOutput, + run as runBenchmark, + serialize as serializeBenchmarkOutput, +} from './benchmark' export type { - ExperimentConfig, - ExperimentModelConfig, - Model, - ModelConfig, - ModelInfo, - ReasoningEffort, - TreatmentConfig, + BenchmarkConfig, + Benchmark, + BenchmarkOutput, + BenchmarkRunResult, + BenchmarkTrialResult, + Capability, +} from './benchmark' + +export { + ExperimentConfigSchema, + defineConfig as defineExperimentConfig, + deserialize as deserializeExperimentOutput, + getExperiment, + listExperiments, + output as getExperimentOutput, + run as runExperiment, + serialize as serializeExperimentOutput, } from './experiment' -export {createAgentEvalOutput, parseAgentEvalOutput} from './output' -export type {AgentEvalOutput, AgentEvalOutputResult} from './output' +export type {ExperimentConfig, Experiment, ExperimentOutput} from './experiment' + +export {defineConfig as defineScenarioConfig, getScenario, listScenarios, ScenarioConfigSchema} from './scenario' +export type {ScenarioConfig, Scenario, ScenarioSourceOptions} from './scenario' + +export {TreatmentSchema, ControlTreatment} from './treatment' +export type {Treatment} from './treatment' + +export {TrialSchema, TrialResultSchema, run as runTrial, compare as compareTrial} from './trial' +export type {Trial, TrialResult} from './trial' diff --git a/packages/agent-eval/src/logger.test.ts b/packages/agent-eval/src/logger.test.ts new file mode 100644 index 00000000..15debe9c --- /dev/null +++ b/packages/agent-eval/src/logger.test.ts @@ -0,0 +1,7 @@ +import {expect, test} from 'vitest' +import {logger} from './logger' + +test('logger uses the default level and is disabled during tests', () => { + expect(logger.level).toBe('silent') + expect(logger.isLevelEnabled('info')).toBe(false) +}) diff --git a/packages/agent-eval/src/logger.ts b/packages/agent-eval/src/logger.ts new file mode 100644 index 00000000..a97ce54b --- /dev/null +++ b/packages/agent-eval/src/logger.ts @@ -0,0 +1,25 @@ +import process from 'node:process' +import pino from 'pino' +import pretty from 'pino-pretty' + +const CI = process.env.CI === 'true' || process.env.CI === '1' || process.env.GITHUB_ACTIONS === 'true' + +const stream = [] + +if (!CI) { + stream.push( + pretty({ + colorize: true, + }), + ) +} + +export const logger = pino( + { + base: undefined, + level: 'info', + timestamp: false, + enabled: process.env.NODE_ENV !== 'test', + }, + ...stream, +) diff --git a/packages/agent-eval/src/mcp-config.test.ts b/packages/agent-eval/src/mcp-config.test.ts new file mode 100644 index 00000000..9692792c --- /dev/null +++ b/packages/agent-eval/src/mcp-config.test.ts @@ -0,0 +1,55 @@ +import {describe, expect, test} from 'vitest' +import {McpConfigFileSchema, McpServerConfigSchema} from './mcp-config' + +describe('McpServerConfigSchema', () => { + test('parses a local MCP server configuration', () => { + expect( + McpServerConfigSchema.parse({ + command: 'npx', + type: 'local', + args: ['example-server'], + env: { + TOKEN: 'token', + }, + tools: ['example'], + }), + ).toEqual({ + command: 'npx', + type: 'local', + args: ['example-server'], + env: { + TOKEN: 'token', + }, + tools: ['example'], + }) + }) + + test('rejects unsupported server types', () => { + expect(() => { + McpServerConfigSchema.parse({ + command: 'https://example.com', + type: 'remote', + }) + }).toThrow() + }) +}) + +test('McpConfigFileSchema parses named servers', () => { + expect( + McpConfigFileSchema.parse({ + mcpServers: { + example: { + command: 'example-server', + type: 'local', + }, + }, + }), + ).toEqual({ + mcpServers: { + example: { + command: 'example-server', + type: 'local', + }, + }, + }) +}) diff --git a/packages/agent-eval/src/model.test.ts b/packages/agent-eval/src/model.test.ts index 6376d2d5..e7b6646b 100644 --- a/packages/agent-eval/src/model.test.ts +++ b/packages/agent-eval/src/model.test.ts @@ -1,48 +1,53 @@ -import {describe, expect, test} from 'vitest' -import {models} from './index' -import {resolveModelConfigs} from './model' +import {test, expect} from 'vitest' +import {getModelVariants} from './model' -test('provides model information', () => { - expect(models).toEqual([ - {name: 'claude-haiku-4.5', reasoningEfforts: []}, - {name: 'claude-opus-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max']}, - {name: 'claude-opus-4.7', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'claude-opus-4.8', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'claude-opus-5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'claude-sonnet-4.5', reasoningEfforts: []}, - {name: 'claude-sonnet-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max']}, - {name: 'claude-sonnet-5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'gemini-3.1-pro-preview', reasoningEfforts: ['low', 'medium', 'high']}, - {name: 'gemini-3.5-flash', reasoningEfforts: ['minimal', 'low', 'medium', 'high']}, - {name: 'gemini-3.6-flash', reasoningEfforts: []}, - {name: 'gpt-5.4', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']}, - {name: 'gpt-5.4-mini', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']}, - {name: 'gpt-5.5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh']}, - {name: 'gpt-5.6-luna', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'gpt-5.6-sol', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, - {name: 'gpt-5.6-terra', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max']}, +test('getModelVariants', () => { + expect(getModelVariants([])).toEqual([]) + + expect(getModelVariants(['claude-opus-5'])).toEqual([ + { + name: 'claude-opus-5', + reasoningEffort: 'medium', + }, + ]) + + expect(getModelVariants([{name: 'claude-opus-5'}])).toEqual([ + { + name: 'claude-opus-5', + reasoningEffort: 'medium', + }, ]) -}) -describe('resolveModelConfigs', () => { - test('resolves each reasoning effort for a model', () => { - expect(resolveModelConfigs({name: 'gpt-5.5', reasoningEfforts: ['low', 'high']})).toEqual([ - { - name: 'gpt-5.5', - reasoningEffort: 'low', - }, - { - name: 'gpt-5.5', - reasoningEffort: 'high', - }, - ]) - }) + expect(getModelVariants([{name: 'claude-opus-5', reasoningEfforts: []}])).toEqual([ + { + name: 'claude-opus-5', + reasoningEffort: 'medium', + }, + ]) + + expect(getModelVariants([{name: 'claude-opus-5', reasoningEfforts: ['medium', 'high']}])).toEqual([ + { + name: 'claude-opus-5', + reasoningEffort: 'medium', + }, + { + name: 'claude-opus-5', + reasoningEffort: 'high', + }, + ]) - test('omits reasoning effort for a model without supported efforts', () => { - expect(resolveModelConfigs({name: 'claude-haiku-4.5', reasoningEfforts: []})).toEqual([ - { - name: 'claude-haiku-4.5', - }, - ]) - }) + expect(getModelVariants(['gpt-5.6-sol', 'gpt-5.6-luna', 'gpt-5.6-terra'])).toEqual([ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + { + name: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }, + { + name: 'gpt-5.6-terra', + reasoningEffort: 'medium', + }, + ]) }) diff --git a/packages/agent-eval/src/model.ts b/packages/agent-eval/src/model.ts index c5ff917e..40381059 100644 --- a/packages/agent-eval/src/model.ts +++ b/packages/agent-eval/src/model.ts @@ -1,8 +1,6 @@ +import * as z from 'zod/mini' + const models = [ - { - name: 'claude-haiku-4.5', - reasoningEfforts: [], - }, { name: 'claude-opus-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max'], @@ -19,10 +17,6 @@ const models = [ name: 'claude-opus-5', reasoningEfforts: ['low', 'medium', 'high', 'xhigh', 'max'], }, - { - name: 'claude-sonnet-4.5', - reasoningEfforts: [], - }, { name: 'claude-sonnet-4.6', reasoningEfforts: ['low', 'medium', 'high', 'max'], @@ -39,10 +33,6 @@ const models = [ name: 'gemini-3.5-flash', reasoningEfforts: ['minimal', 'low', 'medium', 'high'], }, - { - name: 'gemini-3.6-flash', - reasoningEfforts: [], - }, { name: 'gpt-5.4', reasoningEfforts: ['low', 'medium', 'high', 'xhigh'], @@ -69,27 +59,122 @@ const models = [ }, ] as const -type ModelInfo = (typeof models)[number] -type Model = ModelInfo['name'] -type ReasoningEffort = ModelInfo['reasoningEfforts'][number] -type ModelConfig = { - [Info in ModelInfo as Info['name']]: { - name: Info['name'] - reasoningEfforts: Array +const reasoningEffortsByModel = new Map>( + models.map(model => { + return [model.name, new Set(model.reasoningEfforts)] + }), +) + +type ModelConfig = (typeof models)[number] +type Model = ModelConfig['name'] +type ReasoningEfforts = Extract['reasoningEfforts'] +type ReasoningEffort = ReasoningEfforts[number] +type ModelVariant = M extends Model + ? { + name: M + reasoningEffort: ReasoningEffort + } + : never + +const ModelVariantSchema = z.custom(value => { + if (typeof value !== 'object' || value === null) { + return false } -}[Model] -type ExperimentModelConfig = ModelConfig -function resolveModelConfigs(config: ExperimentModelConfig): Array<{name: Model; reasoningEffort?: ReasoningEffort}> { - if (config.reasoningEfforts.length === 0) { - return [{name: config.name}] + if (!('name' in value) || typeof value.name !== 'string') { + return false } - return config.reasoningEfforts.map(reasoningEffort => ({ - name: config.name, - reasoningEffort, - })) + if (!('reasoningEffort' in value) || typeof value.reasoningEffort !== 'string') { + return false + } + + return reasoningEffortsByModel.get(value.name)?.has(value.reasoningEffort) ?? false +}) + +const ModelVariantConfigSchema = z.array( + z.union([ + z.enum(models.map(model => model.name)), + z.object({ + name: z.literal(models[0].name), + reasoningEfforts: z.optional(z.array(z.enum(models[0].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[1].name), + reasoningEfforts: z.optional(z.array(z.enum(models[1].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[2].name), + reasoningEfforts: z.optional(z.array(z.enum(models[2].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[3].name), + reasoningEfforts: z.optional(z.array(z.enum(models[3].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[4].name), + reasoningEfforts: z.optional(z.array(z.enum(models[4].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[5].name), + reasoningEfforts: z.optional(z.array(z.enum(models[5].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[6].name), + reasoningEfforts: z.optional(z.array(z.enum(models[6].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[7].name), + reasoningEfforts: z.optional(z.array(z.enum(models[7].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[8].name), + reasoningEfforts: z.optional(z.array(z.enum(models[8].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[9].name), + reasoningEfforts: z.optional(z.array(z.enum(models[9].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[10].name), + reasoningEfforts: z.optional(z.array(z.enum(models[10].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[11].name), + reasoningEfforts: z.optional(z.array(z.enum(models[11].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[12].name), + reasoningEfforts: z.optional(z.array(z.enum(models[12].reasoningEfforts))), + }), + z.object({ + name: z.literal(models[13].name), + reasoningEfforts: z.optional(z.array(z.enum(models[13].reasoningEfforts))), + }), + ]), +) + +type ModelVariantConfig = z.infer + +function getModelVariants(input: ModelVariantConfig): Array> { + return input.flatMap(config => { + if (typeof config === 'string') { + return [{name: config, reasoningEffort: 'medium'}] + } + + if (!config.reasoningEfforts) { + return [{name: config.name, reasoningEffort: 'medium'}] + } + + if (config.reasoningEfforts.length === 0) { + return [{name: config.name, reasoningEffort: 'medium'}] + } + + return config.reasoningEfforts.map(effort => { + return {name: config.name, reasoningEffort: effort} + }) + }) as Array> } -export {models, resolveModelConfigs} -export type {ExperimentModelConfig, Model, ModelConfig, ModelInfo, ReasoningEffort} +export {models, getModelVariants, ModelVariantSchema, ModelVariantConfigSchema} +export type {ModelConfig, Model, ModelVariant, ReasoningEffort, ModelVariantConfig} diff --git a/packages/agent-eval/src/output.test.ts b/packages/agent-eval/src/output.test.ts deleted file mode 100644 index da378252..00000000 --- a/packages/agent-eval/src/output.test.ts +++ /dev/null @@ -1,239 +0,0 @@ -import {describe, expect, test} from 'vitest' -import { - createAgentEvalOutput, - parseAgentEvalOutput, - type AgentEvalOutput, - type ExperimentConfig, - type ResolvedScenario, - type TreatmentResult, -} from './index' - -const output: AgentEvalOutput = { - id: 'run-id', - experiment: { - id: 'example', - name: 'Example', - description: 'An example experiment', - models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], - scenarios: ['example'], - }, - scenarios: [ - { - id: 'example', - directory: '/scenarios/example', - config: { - description: 'Evaluate whether the agent builds an example', - prompt: 'Build an example', - }, - testPath: '/scenarios/example/scenario.test.ts', - browserTestPath: '/scenarios/example/scenario.browser.test.ts', - }, - ], - treatments: [ - { - id: 'treatment-id', - config: { - name: 'Control', - }, - }, - ], - results: [ - { - id: 'result-id', - treatmentId: 'treatment-id', - model: 'gpt-5.5', - reasoningEffort: 'high', - scenarioId: 'example', - artifacts: { - copilotConfigPath: '/artifacts/.copilot', - directory: '/artifacts', - skillsConfigPath: '/artifacts/.agents', - testResultsPath: '/artifacts/workspace/test-results.json', - workspacePath: '/artifacts/workspace', - }, - assistant: { - logs: [], - turns: 1, - outputTokens: 100, - premiumRequests: 1, - totalApiDurationMs: 1000, - sessionDurationMs: 2000, - tools: { - view: 1, - }, - }, - testResults: { - numTotalTests: 1, - numPassedTests: 1, - numFailedTests: 0, - numPendingTests: 0, - numTodoTests: 0, - tests: [ - { - title: 'renders an example', - fullName: 'example > renders an example', - status: 'passed', - }, - ], - }, - walkthrough: { - type: 'Unavailable', - }, - }, - ], -} - -describe(createAgentEvalOutput, () => { - test('deduplicates experiment, scenario, and treatment metadata', () => { - const experiment: ExperimentConfig = { - name: 'Example', - description: 'An example experiment', - models: [{name: 'gpt-5.5', reasoningEfforts: ['high']}], - scenarios: ['example'], - treatments: [], - } - const scenario: ResolvedScenario = output.scenarios[0] - const result: TreatmentResult = { - ...output.results[0], - treatment: { - config: { - name: 'Control', - }, - scenario, - experiment, - id: 'treatment-id', - model: 'gpt-5.5', - reasoningEffort: 'high', - }, - } - const duplicateTreatmentResult: TreatmentResult = { - ...result, - id: 'second-result-id', - treatment: { - ...result.treatment, - id: 'second-treatment-id', - }, - } - - expect( - createAgentEvalOutput({ - id: 'run-id', - experimentId: 'example', - experiment, - scenarios: [scenario], - results: [result, duplicateTreatmentResult], - }), - ).toEqual({ - ...output, - results: [ - output.results[0], - { - ...output.results[0], - id: 'second-result-id', - }, - ], - }) - }) -}) - -describe(parseAgentEvalOutput, () => { - test('parses agent eval output', () => { - expect(parseAgentEvalOutput(output)).toEqual(output) - }) - - test('parses serialized agent eval output', () => { - expect(parseAgentEvalOutput(JSON.stringify(output))).toEqual(output) - }) - - test('preserves unknown Copilot messages', () => { - const unknownMessage = { - type: 'unknown.event', - data: { - nested: { - value: 42, - }, - }, - metadata: ['one', 'two'], - } - const outputWithUnknownMessage = { - ...output, - results: [ - { - ...output.results[0], - assistant: { - ...output.results[0].assistant, - logs: [unknownMessage], - }, - }, - ], - } - - expect(parseAgentEvalOutput(outputWithUnknownMessage)).toEqual(outputWithUnknownMessage) - }) - - test('defaults the walkthrough for results without one', () => { - const {walkthrough, ...resultWithoutWalkthrough} = output.results[0] - - expect(walkthrough).toEqual({type: 'Unavailable'}) - expect( - parseAgentEvalOutput({ - ...output, - results: [resultWithoutWalkthrough], - }), - ).toEqual(output) - }) - - test('throws for invalid agent eval output', () => { - expect(() => - parseAgentEvalOutput({ - ...output, - results: [ - { - ...output.results[0], - testResults: { - ...output.results[0].testResults, - tests: [{title: 'invalid', fullName: 'invalid', status: 'unknown'}], - }, - }, - ], - }), - ).toThrow() - }) - - test('throws for an invalid reasoning effort', () => { - expect(() => - parseAgentEvalOutput({ - ...output, - results: [ - { - ...output.results[0], - reasoningEffort: 'invalid', - }, - ], - }), - ).toThrow() - }) - - test('throws for an unknown treatment reference', () => { - expect(() => - parseAgentEvalOutput({ - ...output, - results: [ - { - ...output.results[0], - treatmentId: 'unknown', - }, - ], - }), - ).toThrow('references unknown treatment') - }) - - test('throws for duplicate treatment IDs', () => { - expect(() => - parseAgentEvalOutput({ - ...output, - treatments: [...output.treatments, output.treatments[0]], - }), - ).toThrow('Treatment IDs must be unique') - }) -}) diff --git a/packages/agent-eval/src/output.ts b/packages/agent-eval/src/output.ts deleted file mode 100644 index 7844909e..00000000 --- a/packages/agent-eval/src/output.ts +++ /dev/null @@ -1,268 +0,0 @@ -import * as z from 'zod/mini' -import {MessageSchema, type Message} from './copilot-cli' -import type {ExperimentConfig, ExperimentScenarioConfig} from './experiment-config' -import {models} from './model' -import type {Model, ReasoningEffort} from './model' -import type {ResolvedScenario} from './resolve-experiment-scenario' -import type {TreatmentResult, Walkthrough} from './treatment' - -type AgentEvalOutputResult = { - id: string - treatmentId: string - model: Model - reasoningEffort?: ReasoningEffort - scenarioId: string - artifacts: { - copilotConfigPath: string - directory: string - skillsConfigPath: string - testResultsPath: string - workspacePath: string - } - assistant: { - logs: Array - turns: number - outputTokens: number - premiumRequests: number - totalApiDurationMs: number - sessionDurationMs: number - tools: Record - } - testResults: { - numTotalTests: number - numPassedTests: number - numFailedTests: number - numPendingTests: number - numTodoTests: number - tests: Array<{ - title: string - fullName: string - status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' - description?: string - }> - } - walkthrough: Walkthrough -} - -type AgentEvalOutput = { - id: string - experiment: { - id: string - name: string - description: string - models: Array<{ - name: Model - reasoningEfforts: Array - }> - scenarios: Array - } - scenarios: Array - treatments: Array<{ - id: string - config: { - name: string - } - }> - results: Array -} - -const modelNames = new Set(models.map(model => model.name)) -const reasoningEfforts = new Set(models.flatMap(model => model.reasoningEfforts)) -const ModelSchema = z.custom( - value => typeof value === 'string' && modelNames.has(value), - 'Expected a supported model', -) -const ReasoningEffortSchema = z.custom( - value => typeof value === 'string' && reasoningEfforts.has(value), - 'Expected a supported reasoning effort', -) - -const ExperimentScenarioSchema = z.union([ - z.string(), - z.object({ - name: z.optional(z.string()), - path: z.string(), - }), -]) - -const TreatmentConfigSchema = z.object({ - name: z.string(), -}) - -const ExperimentModelConfigSchema = z.object({ - name: ModelSchema, - reasoningEfforts: z.array(ReasoningEffortSchema), -}) - -const AgentEvalOutputExperimentSchema = z.object({ - id: z.string(), - name: z.string(), - description: z.string(), - models: z.array(ExperimentModelConfigSchema), - scenarios: z.array(ExperimentScenarioSchema), -}) - -const ResolvedScenarioSchema = z.object({ - id: z.string(), - directory: z.string(), - config: z.object({ - description: z.optional(z.string()), - prompt: z.string(), - }), - testPath: z.string(), - browserTestPath: z.optional(z.string()), -}) - -const unavailableWalkthrough = {type: 'Unavailable'} as const - -const AgentEvalOutputResultSchema = z.object({ - id: z.string(), - treatmentId: z.string(), - model: ModelSchema, - reasoningEffort: z.optional(ReasoningEffortSchema), - scenarioId: z.string(), - artifacts: z.object({ - copilotConfigPath: z.string(), - directory: z.string(), - skillsConfigPath: z.string(), - testResultsPath: z.string(), - workspacePath: z.string(), - }), - assistant: z.object({ - logs: z.array(MessageSchema), - turns: z.number(), - outputTokens: z.number(), - premiumRequests: z.number(), - totalApiDurationMs: z.number(), - sessionDurationMs: z.number(), - tools: z.record(z.string(), z.number()), - }), - testResults: z.object({ - numTotalTests: z.number(), - numPassedTests: z.number(), - numFailedTests: z.number(), - numPendingTests: z.number(), - numTodoTests: z.number(), - tests: z.array( - z.object({ - title: z.string(), - fullName: z.string(), - status: z.enum(['passed', 'failed', 'skipped', 'pending', 'todo', 'disabled']), - description: z.optional(z.string()), - }), - ), - }), - // Runs created before walkthroughs were supported do not include this field - walkthrough: z.pipe( - z.optional( - z.discriminatedUnion('type', [ - z.object({type: z.literal('Unavailable')}), - z.object({type: z.literal('Screenshot'), filepath: z.string()}), - z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}), - z.object({type: z.literal('Video'), filepath: z.string()}), - ]), - ), - z.transform(value => value ?? unavailableWalkthrough), - ), -}) - -const AgentEvalOutputSchema = z.object({ - id: z.string(), - experiment: AgentEvalOutputExperimentSchema, - scenarios: z.array(ResolvedScenarioSchema), - treatments: z.array( - z.object({ - id: z.string(), - config: TreatmentConfigSchema, - }), - ), - results: z.array(AgentEvalOutputResultSchema), -}) - -type CreateAgentEvalOutputOptions = { - id: string - experimentId: string - experiment: ExperimentConfig - scenarios: Array - results: Array -} - -function createAgentEvalOutput({ - id, - experimentId, - experiment, - scenarios, - results, -}: CreateAgentEvalOutputOptions): AgentEvalOutput { - const treatmentsByName = new Map< - string, - { - id: string - config: { - name: string - } - } - >() - - for (const result of results) { - if (!treatmentsByName.has(result.treatment.config.name)) { - treatmentsByName.set(result.treatment.config.name, { - id: result.treatment.id, - config: { - name: result.treatment.config.name, - }, - }) - } - } - - return { - id, - experiment: { - id: experimentId, - name: experiment.name, - description: experiment.description, - models: experiment.models, - scenarios: experiment.scenarios, - }, - scenarios, - treatments: [...treatmentsByName.values()], - results: results.map(result => { - const treatment = treatmentsByName.get(result.treatment.config.name) - if (!treatment) { - throw new Error(`Treatment "${result.treatment.config.name}" was not normalized`) - } - - return { - id: result.id, - treatmentId: treatment.id, - model: result.treatment.model, - reasoningEffort: result.treatment.reasoningEffort, - scenarioId: result.treatment.scenario.id, - artifacts: result.artifacts, - assistant: result.assistant, - testResults: result.testResults, - walkthrough: result.walkthrough, - } - }), - } -} - -function parseAgentEvalOutput(value: unknown): AgentEvalOutput { - const input = typeof value === 'string' ? JSON.parse(value) : value - const output = AgentEvalOutputSchema.parse(input, {reportInput: true}) - const treatmentIds = new Set(output.treatments.map(treatment => treatment.id)) - if (treatmentIds.size !== output.treatments.length) { - throw new Error('Treatment IDs must be unique') - } - - for (const result of output.results) { - if (!treatmentIds.has(result.treatmentId)) { - throw new Error(`Result "${result.id}" references unknown treatment "${result.treatmentId}"`) - } - } - - return output -} - -export {createAgentEvalOutput, parseAgentEvalOutput} -export type {AgentEvalOutput, AgentEvalOutputResult} diff --git a/packages/agent-eval/src/plan.test.ts b/packages/agent-eval/src/plan.test.ts new file mode 100644 index 00000000..f85de6f8 --- /dev/null +++ b/packages/agent-eval/src/plan.test.ts @@ -0,0 +1,168 @@ +import {afterEach, describe, expect, test, vi} from 'vitest' +import {VirtualHost} from './host' +import {create, run} from './plan' +import {run as runTrial} from './trial' +import type {Trial, TrialResult} from './trial' + +vi.mock('./trial', async importOriginal => { + const original = await importOriginal() + return { + ...original, + run: vi.fn(), + } +}) + +afterEach(() => { + vi.clearAllMocks() + vi.restoreAllMocks() +}) + +function createTrial(id: string): Trial { + return { + id, + scenario: { + id: 'scenario', + directory: '/scenario', + prompt: 'prompt', + tags: [], + testPath: '/scenario/scenario.test.ts', + }, + treatment: { + name: 'Control', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } +} + +function createResult(trial: Trial): TrialResult { + return { + artifacts: { + directory: '/artifacts', + copilotConfigDirectory: '/artifacts/.copilot', + skillsConfigDirectory: '/artifacts/.agents', + testResultsPath: '/artifacts/test-results.json', + workspaceDirectory: '/artifacts/workspace', + }, + trial, + agent: { + sessions: [], + }, + testResults: { + numTotalTests: 0, + numPassedTests: 0, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }, + walkthrough: { + type: 'Unavailable', + }, + } +} + +describe('create', () => { + test('randomizes trials without mutating the input', async () => { + const trials = [createTrial('one'), createTrial('two'), createTrial('three')] + vi.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(0) + + const plan = await create(trials) + + expect(plan.trials.map(trial => trial.id)).toEqual(['two', 'three', 'one']) + expect(trials.map(trial => trial.id)).toEqual(['one', 'two', 'three']) + }) +}) + +describe('run', () => { + test('runs each trial and returns results in plan order', async () => { + const trials = [createTrial('one'), createTrial('two')] + const results = trials.map(createResult) + const host = VirtualHost.create() + vi.mocked(runTrial).mockImplementation(async ({trial}) => { + return createResult(trial) + }) + + await expect( + run({ + env: { + artifactsDirectory: '/artifacts', + benchmarksDirectory: '/benchmarks', + concurrency: 2, + copilotToken: 'token', + dockerImage: 'node:test', + experimentsDirectory: '/experiments', + outputPath: '/output.json', + scenariosDirectory: '/scenarios', + }, + host, + plan: { + trials, + }, + }), + ).resolves.toEqual(results) + expect(runTrial).toHaveBeenCalledTimes(2) + }) + + test('retries a failed trial three times', async () => { + const trial = createTrial('one') + const result = createResult(trial) + const host = VirtualHost.create() + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.mocked(runTrial) + .mockRejectedValueOnce(new Error('first')) + .mockRejectedValueOnce(new Error('second')) + .mockRejectedValueOnce(new Error('third')) + .mockResolvedValueOnce(result) + + await expect( + run({ + env: { + artifactsDirectory: '/artifacts', + benchmarksDirectory: '/benchmarks', + concurrency: 1, + copilotToken: 'token', + dockerImage: 'node:test', + experimentsDirectory: '/experiments', + outputPath: '/output.json', + scenariosDirectory: '/scenarios', + }, + host, + plan: { + trials: [trial], + }, + }), + ).resolves.toEqual([result]) + expect(runTrial).toHaveBeenCalledTimes(4) + }) + + test('throws after all retry attempts fail', async () => { + const host = VirtualHost.create() + const error = new Error('failure') + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.mocked(runTrial).mockRejectedValue(error) + + await expect( + run({ + env: { + artifactsDirectory: '/artifacts', + benchmarksDirectory: '/benchmarks', + concurrency: 1, + copilotToken: 'token', + dockerImage: 'node:test', + experimentsDirectory: '/experiments', + outputPath: '/output.json', + scenariosDirectory: '/scenarios', + }, + host, + plan: { + trials: [createTrial('one')], + }, + }), + ).rejects.toBe(error) + expect(runTrial).toHaveBeenCalledTimes(4) + }) +}) diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts new file mode 100644 index 00000000..9d503cc8 --- /dev/null +++ b/packages/agent-eval/src/plan.ts @@ -0,0 +1,80 @@ +import Queue from 'p-queue' +import {run as runTrial} from './trial' +import type {Trial, TrialResult} from './trial' +import type {EnvironmentConfig} from './environment' +import {DefaultHost, type Host} from './host' +import {logger} from './logger' + +/** + * A plan is an ordered list of trials to be ran. + */ +type Plan = { + trials: Array +} + +// TODO: support plan with sharding +async function create(trials: Array): Promise { + return { + trials: randomize(trials), + } +} + +function randomize(input: Array): Array { + const randomized: Array = input.slice() + + // Fisher–Yates shuffle + for (let i = randomized.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]] + } + + return randomized +} + +type RunPlanOptions = { + env: EnvironmentConfig + host?: Host + plan: Plan +} + +async function run({env, host = DefaultHost, plan}: RunPlanOptions): Promise> { + const queue = new Queue({ + concurrency: env.concurrency, + }) + + const results = await Promise.all( + plan.trials.map(trial => { + return queue.add(() => { + return retry(async () => { + await using sandbox = await host.createSandbox({ + dockerImage: env.dockerImage, + }) + return await runTrial({ + artifactsDirectory: env.artifactsDirectory, + copilotToken: env.copilotToken, + host, + sandbox, + trial, + }) + }) + }) + }), + ) + + return results +} + +async function retry(fn: () => Promise, retries: number = 3): Promise { + try { + return await fn() + } catch (error) { + if (retries > 0) { + logger.error({error}, 'Retrying') + return retry(fn, retries - 1) + } + throw error + } +} + +export {create, run} +export type {Plan} diff --git a/packages/agent-eval/src/report.test.ts b/packages/agent-eval/src/report.test.ts new file mode 100644 index 00000000..c99317e1 --- /dev/null +++ b/packages/agent-eval/src/report.test.ts @@ -0,0 +1,381 @@ +import {expect, test} from 'vitest' +import type {BenchmarkTrialResult, Capability} from './benchmark' +import type {TrialResult} from './trial' +import {formatBenchmarkResults, formatExperimentResults} from './report' + +function createResult({ + treatment, + scenario, + model, + numPassedTests, + numTotalTests, + sessions, +}: { + treatment: string + scenario: string + model: TrialResult['trial']['model'] + numPassedTests: number + numTotalTests: number + sessions: TrialResult['agent']['sessions'] +}): TrialResult { + return { + artifacts: { + directory: '/artifacts/trial', + copilotConfigDirectory: '/artifacts/trial/.copilot', + skillsConfigDirectory: '/artifacts/trial/.agents', + testResultsPath: '/artifacts/trial/workspace/test-results.json', + workspaceDirectory: '/artifacts/trial/workspace', + }, + trial: { + id: `${treatment}-${scenario}-${model.name}-${model.reasoningEffort}`, + scenario: { + id: scenario, + directory: `/scenarios/${scenario}`, + prompt: 'Complete the task', + tags: [], + testPath: `/scenarios/${scenario}/scenario.test.ts`, + }, + treatment: { + name: treatment, + }, + model, + }, + agent: { + sessions, + }, + testResults: { + numTotalTests, + numPassedTests, + numFailedTests: numTotalTests - numPassedTests, + numPendingTests: 0, + numTodoTests: 0, + success: numPassedTests === numTotalTests, + testResults: [], + }, + walkthrough: { + type: 'Unavailable', + }, + } +} + +test('formats trial results as an aggregated hierarchy', () => { + const results = [ + createResult({ + treatment: 'Control', + scenario: 'button', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + numPassedTests: 1, + numTotalTests: 2, + sessions: [ + { + turns: 2, + outputTokens: 1_000, + premiumRequests: 1, + totalApiDurationMs: 30_000, + sessionDurationMs: 45_000, + tools: {}, + messages: [], + }, + { + turns: 1, + outputTokens: 250, + premiumRequests: 2, + totalApiDurationMs: 5_000, + sessionDurationMs: 10_000, + tools: {}, + messages: [], + }, + ], + }), + createResult({ + treatment: 'Recommended', + scenario: 'button', + model: { + name: 'claude-sonnet-5', + reasoningEffort: 'high', + }, + numPassedTests: 2, + numTotalTests: 2, + sessions: [ + { + turns: 1, + outputTokens: 900, + premiumRequests: 1, + totalApiDurationMs: 65_000, + sessionDurationMs: 90_000, + tools: {}, + messages: [], + }, + ], + }), + ] + + expect(formatExperimentResults('Design system', results)).toMatchInlineSnapshot(` + "Experiment Treatment Scenario Model Reasoning Effort Success Rate Tests Runs Output Tokens Premium Requests Session Time API Time + ------------- ----------- ------------- ------------------- ---------------- ------------ ----- ---- ------------- ---------------- ------------ -------- + Design system Recommended All scenarios All models 100.0% 2/2 1 900 1 1m 30.0s 1m 5.0s + button All models 100.0% 2/2 1 900 1 1m 30.0s 1m 5.0s + claude-sonnet-5 high 100.0% 2/2 1 900 1 1m 30.0s 1m 5.0s + Design system Control All scenarios All models 50.0% 1/2 1 1,250 3 55.0s 35.0s + button All models 50.0% 1/2 1 1,250 3 55.0s 35.0s + gpt-5.6-sol medium 50.0% 1/2 1 1,250 3 55.0s 35.0s " + `) +}) + +test('formats benchmark results as capability comparisons by scenario', () => { + const capabilities: Array = [ + { + name: 'Migration', + scenarios: [ + { + id: 'migrate-component', + directory: '/scenarios/migrate-component', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/migrate-component/scenario.test.ts', + }, + ], + }, + { + name: 'Authoring', + scenarios: [ + { + id: 'create-component', + directory: '/scenarios/create-component', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/create-component/scenario.test.ts', + }, + ], + }, + ] + const results: Array = [ + { + ...createResult({ + treatment: 'Control', + scenario: 'create-component', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + numPassedTests: 1, + numTotalTests: 2, + sessions: [ + { + turns: 2, + outputTokens: 1_000, + premiumRequests: 1, + totalApiDurationMs: 30_000, + sessionDurationMs: 45_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[1], + }, + { + ...createResult({ + treatment: 'Benchmark', + scenario: 'create-component', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + numPassedTests: 2, + numTotalTests: 2, + sessions: [ + { + turns: 1, + outputTokens: 800, + premiumRequests: 2, + totalApiDurationMs: 20_000, + sessionDurationMs: 35_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[1], + }, + { + ...createResult({ + treatment: 'Control', + scenario: 'create-component', + model: { + name: 'claude-sonnet-5', + reasoningEffort: 'high', + }, + numPassedTests: 0, + numTotalTests: 2, + sessions: [ + { + turns: 1, + outputTokens: 700, + premiumRequests: 1, + totalApiDurationMs: 20_000, + sessionDurationMs: 30_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[1], + }, + { + ...createResult({ + treatment: 'Benchmark', + scenario: 'create-component', + model: { + name: 'claude-sonnet-5', + reasoningEffort: 'high', + }, + numPassedTests: 0, + numTotalTests: 2, + sessions: [ + { + turns: 1, + outputTokens: 700, + premiumRequests: 1, + totalApiDurationMs: 15_000, + sessionDurationMs: 25_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[1], + }, + { + ...createResult({ + treatment: 'Control', + scenario: 'migrate-component', + model: { + name: 'claude-sonnet-5', + reasoningEffort: 'high', + }, + numPassedTests: 2, + numTotalTests: 2, + sessions: [ + { + turns: 2, + outputTokens: 900, + premiumRequests: 2, + totalApiDurationMs: 40_000, + sessionDurationMs: 60_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[0], + }, + { + ...createResult({ + treatment: 'Benchmark', + scenario: 'migrate-component', + model: { + name: 'claude-sonnet-5', + reasoningEffort: 'high', + }, + numPassedTests: 1, + numTotalTests: 2, + sessions: [ + { + turns: 3, + outputTokens: 1_100, + premiumRequests: 3, + totalApiDurationMs: 50_000, + sessionDurationMs: 75_000, + tools: {}, + messages: [], + }, + ], + }), + capability: capabilities[0], + }, + ] + + const formatted = formatBenchmarkResults( + { + name: 'Design system', + capabilities, + }, + results, + ) + .split('\n') + .map(line => { + return line.trimEnd() + }) + .join('\n') + + expect(formatted).toMatchInlineSnapshot(` + "Benchmark Capability Scenario Model Reasoning Effort Tests Output Tokens Premium Requests Session Time API Time + ------------- ---------- ------------------- ------------------- ---------------- ------------- -------------- ---------------- ----------------- -------------- + Design system Migration All scenarios All models 1/2 (-50.0%) 1,100 (+22.2%) 3 (+50.0%) 1m 15.0s (+25.0%) 50.0s (+25.0%) + migrate-component All models 1/2 (-50.0%) 1,100 (+22.2%) 3 (+50.0%) 1m 15.0s (+25.0%) 50.0s (+25.0%) + claude-sonnet-5 high 1/2 (-50.0%) 1,100 (+22.2%) 3 (+50.0%) 1m 15.0s (+25.0%) 50.0s (+25.0%) + Design system Authoring All scenarios All models 2/4 (+100.0%) 1,500 (-11.8%) 3 (+50.0%) 1m 0.0s (-20.0%) 35.0s (-30.0%) + create-component All models 2/4 (+100.0%) 1,500 (-11.8%) 3 (+50.0%) 1m 0.0s (-20.0%) 35.0s (-30.0%) + gpt-5.6-sol medium 2/2 (+100.0%) 800 (-20.0%) 2 (+100.0%) 35.0s (-22.2%) 20.0s (-33.3%) + claude-sonnet-5 high 0/2 (0%) 700 (0%) 1 (0%) 25.0s (-16.7%) 15.0s (-25.0%)" + `) +}) + +test('compares benchmark test success rates when test totals differ', () => { + const capability: Capability = { + name: 'Authoring', + scenarios: [ + { + id: 'create-component', + directory: '/scenarios/create-component', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/create-component/scenario.test.ts', + }, + ], + } + const results: Array = [ + { + ...createResult({ + treatment: 'Control', + scenario: 'create-component', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + numPassedTests: 1, + numTotalTests: 2, + sessions: [], + }), + capability, + }, + { + ...createResult({ + treatment: 'Benchmark', + scenario: 'create-component', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + numPassedTests: 2, + numTotalTests: 4, + sessions: [], + }), + capability, + }, + ] + + expect( + formatBenchmarkResults( + { + name: 'Design system', + capabilities: [capability], + }, + results, + ), + ).toContain('2/4 (0%)') +}) diff --git a/packages/agent-eval/src/report.ts b/packages/agent-eval/src/report.ts new file mode 100644 index 00000000..c6bdd9ba --- /dev/null +++ b/packages/agent-eval/src/report.ts @@ -0,0 +1,458 @@ +import type {Model, ReasoningEffort} from './model' +import type {Benchmark, BenchmarkTrialResult} from './benchmark' +import type {TrialResult} from './trial' + +type ResultSummary = { + experiment: string + treatment?: string + scenario?: string + model?: Model + reasoningEffort?: ReasoningEffort + runs: number + numPassedTests: number + numTotalTests: number + outputTokens: number + premiumRequests: number + sessionDurationMs: number + totalApiDurationMs: number +} + +type ResultSummaryValues = { + treatment?: string + scenario?: string + model?: Model + reasoningEffort?: ReasoningEffort +} + +type ResultHierarchy = Array<{ + experiment: string + treatments: Array<{ + summary: ResultSummary + scenarios: Array<{ + summary: ResultSummary + models: Array + }> + }> +}> + +type TableRow = Record + +type BenchmarkComparison = { + benchmark: string + capability: string + scenario?: string + model?: Model + reasoningEffort?: ReasoningEffort + control: ResultSummary + benchmarkTreatment: ResultSummary +} + +function createResultSummary(experiment: string, values: ResultSummaryValues = {}): ResultSummary { + return { + experiment, + treatment: values.treatment, + scenario: values.scenario, + model: values.model, + reasoningEffort: values.reasoningEffort, + runs: 0, + numPassedTests: 0, + numTotalTests: 0, + outputTokens: 0, + premiumRequests: 0, + sessionDurationMs: 0, + totalApiDurationMs: 0, + } +} + +function addResultToSummary(summary: ResultSummary, result: TrialResult): void { + summary.runs += 1 + summary.numPassedTests += result.testResults.numPassedTests + summary.numTotalTests += result.testResults.numTotalTests + + for (const session of result.agent.sessions) { + summary.outputTokens += session.outputTokens + summary.premiumRequests += session.premiumRequests + summary.sessionDurationMs += session.sessionDurationMs + summary.totalApiDurationMs += session.totalApiDurationMs + } +} + +function getSuccessRate(summary: ResultSummary): number { + if (summary.numTotalTests === 0) { + return 0 + } + + return summary.numPassedTests / summary.numTotalTests +} + +function compareSummaries(a: ResultSummary, b: ResultSummary): number { + return ( + getSuccessRate(b) - getSuccessRate(a) || + a.outputTokens - b.outputTokens || + a.sessionDurationMs - b.sessionDurationMs || + a.premiumRequests - b.premiumRequests || + a.experiment.localeCompare(b.experiment) || + (a.treatment ?? '').localeCompare(b.treatment ?? '') || + (a.scenario ?? '').localeCompare(b.scenario ?? '') || + (a.model ?? '').localeCompare(b.model ?? '') || + (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '') + ) +} + +function getSummaryKey(experiment: string, values: ResultSummaryValues = {}): string { + return [ + experiment, + values.treatment ?? '', + values.scenario ?? '', + values.model ?? '', + values.reasoningEffort ?? '', + ].join('\0') +} + +function getResultSummaries(experiment: string, results: Array): ResultHierarchy { + const treatmentSummaries = new Map() + const scenarioSummaries = new Map() + const modelSummaries = new Map() + + for (const result of results) { + const treatmentValues = { + treatment: result.trial.treatment.name, + } + const treatmentKey = getSummaryKey(experiment, treatmentValues) + const treatmentSummary = treatmentSummaries.get(treatmentKey) ?? createResultSummary(experiment, treatmentValues) + addResultToSummary(treatmentSummary, result) + treatmentSummaries.set(treatmentKey, treatmentSummary) + + const scenarioValues = { + treatment: result.trial.treatment.name, + scenario: result.trial.scenario.id, + } + const scenarioKey = getSummaryKey(experiment, scenarioValues) + const scenarioSummary = scenarioSummaries.get(scenarioKey) ?? createResultSummary(experiment, scenarioValues) + addResultToSummary(scenarioSummary, result) + scenarioSummaries.set(scenarioKey, scenarioSummary) + + const modelValues = { + treatment: result.trial.treatment.name, + scenario: result.trial.scenario.id, + model: result.trial.model.name, + reasoningEffort: result.trial.model.reasoningEffort, + } + const modelKey = getSummaryKey(experiment, modelValues) + const modelSummary = modelSummaries.get(modelKey) ?? createResultSummary(experiment, modelValues) + addResultToSummary(modelSummary, result) + modelSummaries.set(modelKey, modelSummary) + } + + return [ + { + experiment, + treatments: [...treatmentSummaries.values()].toSorted(compareSummaries).map(summary => { + return { + summary, + scenarios: [...scenarioSummaries.values()] + .filter(scenarioSummary => { + return scenarioSummary.treatment === summary.treatment + }) + .toSorted(compareSummaries) + .map(scenarioSummary => { + return { + summary: scenarioSummary, + models: [...modelSummaries.values()] + .filter(modelSummary => { + return ( + modelSummary.treatment === summary.treatment && modelSummary.scenario === scenarioSummary.scenario + ) + }) + .toSorted(compareSummaries), + } + }), + } + }), + }, + ] +} + +function formatPercent(value: number): string { + return `${(value * 100).toFixed(1)}%` +} + +function formatDuration(ms: number): string { + const seconds = ms / 1000 + + if (seconds < 60) { + return `${seconds.toFixed(1)}s` + } + + const minutes = Math.floor(seconds / 60) + const remainingSeconds = seconds - minutes * 60 + return `${minutes}m ${remainingSeconds.toFixed(1)}s` +} + +function formatNumber(value: number): string { + return new Intl.NumberFormat('en-US').format(value) +} + +function formatTable(rows: Array, columns: Array): string { + const columnWidths = columns.map(column => { + let width = column.length + + for (const row of rows) { + width = Math.max(width, String(row[column] ?? '').length) + } + + return width + }) + + const formatRow = (row: TableRow): string => { + return columns + .map((column, index) => { + return String(row[column] ?? '').padEnd(columnWidths[index]) + }) + .join(' ') + } + + return [ + formatRow(Object.fromEntries(columns.map(column => [column, column]))), + columnWidths.map(width => '-'.repeat(width)).join(' '), + ...rows.map(formatRow), + ].join('\n') +} + +function formatSummaryRow(summary: ResultSummary, level: 'treatment' | 'scenario' | 'model'): TableRow { + return { + Experiment: level === 'treatment' ? summary.experiment : '', + Treatment: level === 'treatment' ? (summary.treatment ?? '') : '', + Scenario: level === 'treatment' ? 'All scenarios' : level === 'scenario' ? ` ${summary.scenario ?? ''}` : '', + Model: level === 'model' ? ` ${summary.model ?? ''}` : 'All models', + 'Reasoning Effort': level === 'model' ? (summary.reasoningEffort ?? '') : '', + 'Success Rate': formatPercent(getSuccessRate(summary)), + Tests: `${summary.numPassedTests}/${summary.numTotalTests}`, + Runs: summary.runs, + 'Output Tokens': formatNumber(summary.outputTokens), + 'Premium Requests': formatNumber(summary.premiumRequests), + 'Session Time': formatDuration(summary.sessionDurationMs), + 'API Time': formatDuration(summary.totalApiDurationMs), + } +} + +function formatPercentDelta(control: number, benchmark: number): string { + if (control === benchmark) { + return '0%' + } + + if (control === 0) { + return 'N/A' + } + + const delta = (benchmark - control) / control + const sign = delta > 0 ? '+' : '' + return `${sign}${(delta * 100).toFixed(1)}%` +} + +function formatBenchmarkValue(value: string, control: number, benchmark: number): string { + return `${value} (${formatPercentDelta(control, benchmark)})` +} + +function compareBenchmarkModelPerformance(a: ResultSummary, b: ResultSummary): number { + return ( + getSuccessRate(b) - getSuccessRate(a) || + a.outputTokens - b.outputTokens || + a.premiumRequests - b.premiumRequests || + a.sessionDurationMs - b.sessionDurationMs || + a.totalApiDurationMs - b.totalApiDurationMs + ) +} + +function getBenchmarkComparisons( + benchmark: Pick, + results: Array, +): Array { + const comparisons = new Map() + + for (const result of results) { + const values = [ + { + key: result.capability.name, + scenario: undefined, + }, + { + key: `${result.capability.name}\0${result.trial.scenario.id}`, + scenario: result.trial.scenario.id, + }, + { + key: [ + result.capability.name, + result.trial.scenario.id, + result.trial.model.name, + result.trial.model.reasoningEffort, + ].join('\0'), + scenario: result.trial.scenario.id, + model: result.trial.model.name, + reasoningEffort: result.trial.model.reasoningEffort, + }, + ] + + for (const value of values) { + const comparison = comparisons.get(value.key) ?? { + benchmark: benchmark.name, + capability: result.capability.name, + scenario: value.scenario, + model: value.model, + reasoningEffort: value.reasoningEffort, + control: createResultSummary(benchmark.name), + benchmarkTreatment: createResultSummary(benchmark.name), + } + const summary = result.trial.treatment.name === 'Control' ? comparison.control : comparison.benchmarkTreatment + addResultToSummary(summary, result) + comparisons.set(value.key, comparison) + } + } + + const capabilityOrder = new Map( + benchmark.capabilities.map((capability, index) => { + return [capability.name, index] + }), + ) + const scenarioOrder = new Map( + benchmark.capabilities.map(capability => { + return [ + capability.name, + new Map( + capability.scenarios.map((scenario, index) => { + return [scenario.id, index] + }), + ), + ] + }), + ) + + return [...comparisons.values()].toSorted((a, b) => { + const capabilityDifference = + (capabilityOrder.get(a.capability) ?? Number.MAX_SAFE_INTEGER) - + (capabilityOrder.get(b.capability) ?? Number.MAX_SAFE_INTEGER) + if (capabilityDifference !== 0) { + return capabilityDifference + } + + const scenarioDifference = + Number(Boolean(a.scenario)) - Number(Boolean(b.scenario)) || + (scenarioOrder.get(a.capability)?.get(a.scenario ?? '') ?? Number.MAX_SAFE_INTEGER) - + (scenarioOrder.get(b.capability)?.get(b.scenario ?? '') ?? Number.MAX_SAFE_INTEGER) || + (a.scenario ?? '').localeCompare(b.scenario ?? '') + if (scenarioDifference !== 0) { + return scenarioDifference + } + + const modelDifference = Number(Boolean(a.model)) - Number(Boolean(b.model)) + if (modelDifference !== 0) { + return modelDifference + } + + if (a.model && b.model) { + return ( + compareBenchmarkModelPerformance(a.benchmarkTreatment, b.benchmarkTreatment) || + a.model.localeCompare(b.model) || + (a.reasoningEffort ?? '').localeCompare(b.reasoningEffort ?? '') + ) + } + + return ( + a.capability.localeCompare(b.capability) || + (a.scenario ?? '').localeCompare(b.scenario ?? '') || + (a.model ?? '').localeCompare(b.model ?? '') + ) + }) +} + +function formatBenchmarkComparison(comparison: BenchmarkComparison): TableRow { + return { + Benchmark: comparison.scenario ? '' : comparison.benchmark, + Capability: comparison.scenario ? '' : comparison.capability, + Scenario: comparison.model ? '' : comparison.scenario ? ` ${comparison.scenario}` : 'All scenarios', + Model: comparison.model ? ` ${comparison.model}` : 'All models', + 'Reasoning Effort': comparison.reasoningEffort ?? '', + Tests: formatBenchmarkValue( + `${comparison.benchmarkTreatment.numPassedTests}/${comparison.benchmarkTreatment.numTotalTests}`, + getSuccessRate(comparison.control), + getSuccessRate(comparison.benchmarkTreatment), + ), + 'Output Tokens': formatBenchmarkValue( + formatNumber(comparison.benchmarkTreatment.outputTokens), + comparison.control.outputTokens, + comparison.benchmarkTreatment.outputTokens, + ), + 'Premium Requests': formatBenchmarkValue( + formatNumber(comparison.benchmarkTreatment.premiumRequests), + comparison.control.premiumRequests, + comparison.benchmarkTreatment.premiumRequests, + ), + 'Session Time': formatBenchmarkValue( + formatDuration(comparison.benchmarkTreatment.sessionDurationMs), + comparison.control.sessionDurationMs, + comparison.benchmarkTreatment.sessionDurationMs, + ), + 'API Time': formatBenchmarkValue( + formatDuration(comparison.benchmarkTreatment.totalApiDurationMs), + comparison.control.totalApiDurationMs, + comparison.benchmarkTreatment.totalApiDurationMs, + ), + } +} + +function formatBenchmarkResults( + benchmark: Pick, + results: Array, +): string { + const columns = [ + 'Benchmark', + 'Capability', + 'Scenario', + 'Model', + 'Reasoning Effort', + 'Tests', + 'Output Tokens', + 'Premium Requests', + 'Session Time', + 'API Time', + ] + const rows = getBenchmarkComparisons(benchmark, results).map(formatBenchmarkComparison) + + return formatTable(rows, columns) +} + +function formatExperimentResults(experiment: string, results: Array): string { + const columns = [ + 'Experiment', + 'Treatment', + 'Scenario', + 'Model', + 'Reasoning Effort', + 'Success Rate', + 'Tests', + 'Runs', + 'Output Tokens', + 'Premium Requests', + 'Session Time', + 'API Time', + ] + const rows: Array = [] + + for (const {treatments} of getResultSummaries(experiment, results)) { + for (const {summary, scenarios} of treatments) { + rows.push(formatSummaryRow(summary, 'treatment')) + + for (const {summary: scenarioSummary, models} of scenarios) { + rows.push(formatSummaryRow(scenarioSummary, 'scenario')) + + for (const model of models) { + rows.push(formatSummaryRow(model, 'model')) + } + } + } + } + + return formatTable(rows, columns) +} + +export {formatBenchmarkResults, formatExperimentResults} diff --git a/packages/agent-eval/src/resolve-experiment-scenario.ts b/packages/agent-eval/src/resolve-experiment-scenario.ts deleted file mode 100644 index 61b22ccd..00000000 --- a/packages/agent-eval/src/resolve-experiment-scenario.ts +++ /dev/null @@ -1,32 +0,0 @@ -import path from 'node:path' -import type {ExperimentScenarioConfig} from './experiment-config' -import {findScenario, loadScenarioDirectory, type ResolvedScenario, type ScenarioSourceOptions} from './scenarios' - -type ResolveScenarioOptions = ScenarioSourceOptions & { - cwd?: string -} - -async function resolveExperimentScenario( - scenarioConfig: ExperimentScenarioConfig, - options: ResolveScenarioOptions, -): Promise { - if (typeof scenarioConfig === 'string') { - const scenario = await findScenario(scenarioConfig, { - directory: options.directory, - }) - if (!scenario) { - throw new Error( - `Scenario "${scenarioConfig}" was not found in: ${path.resolve(options.directory ?? 'scenarios')}`, - ) - } - return scenario - } - - const cwd = options.cwd ?? process.cwd() - const directory = path.resolve(cwd, scenarioConfig.path) - const name = scenarioConfig.name ?? path.basename(directory) - return loadScenarioDirectory(directory, name) -} - -export {resolveExperimentScenario} -export type {ResolvedScenario} from './scenarios' diff --git a/packages/agent-eval/src/run.test.ts b/packages/agent-eval/src/run.test.ts deleted file mode 100644 index fb9abf6a..00000000 --- a/packages/agent-eval/src/run.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import {describe, expect, test} from 'vitest' -import {getCopilotArgs, getVitestConfig} from './run' - -describe('getCopilotArgs', () => { - test('omits reasoning effort when not configured', () => { - expect( - getCopilotArgs({ - prompt: 'Update the page', - model: 'claude-haiku-4.5', - }), - ).not.toContain('--reasoning-effort') - }) - - test('forwards the model and reasoning effort', () => { - expect( - getCopilotArgs({ - prompt: 'Update the page', - model: 'gpt-5.5', - reasoningEffort: 'medium', - }), - ).toEqual([ - '-p', - 'Update the page', - '--model', - 'gpt-5.5', - '--allow-all', - '--reasoning-effort', - 'medium', - '--mode', - 'autopilot', - '--output-format', - 'json', - ]) - }) -}) - -describe('getVitestConfig', () => { - test('configures node tests by default', () => { - const config = getVitestConfig('test-results.json') - - expect(config).toContain(`outputFile: "test-results.json"`) - expect(config).not.toContain('@vitest/browser-playwright') - }) - - test('configures browser tests with Playwright and Chromium', () => { - const config = getVitestConfig('browser-test-results.json', true) - - expect(config).toContain(`import {playwright} from '@vitest/browser-playwright'`) - expect(config).toContain('enabled: true') - expect(config).toContain('headless: true') - expect(config).toContain('provider: playwright()') - expect(config).toContain(`instances: [{browser: 'chromium'}]`) - expect(config).toContain(`outputFile: "browser-test-results.json"`) - }) -}) diff --git a/packages/agent-eval/src/run.ts b/packages/agent-eval/src/run.ts deleted file mode 100644 index a5bc9892..00000000 --- a/packages/agent-eval/src/run.ts +++ /dev/null @@ -1,487 +0,0 @@ -import {randomUUID} from 'node:crypto' -import path from 'node:path' -import fs from 'node:fs/promises' -import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, Sandbox} from './sandbox' -import type {Treatment, TreatmentResult, Walkthrough} from './treatment' -import type {Model, ReasoningEffort} from './model' -import {isMessageType, parseMessage, type Message} from './copilot-cli' -import {getTestMetadata, parseTestResults} from './vitest' -import {existsSync} from 'node:fs' - -const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright' -const CHROMIUM_EXECUTABLE_PATH = '/usr/bin/chromium' -const WALKTHROUGH_DIR = 'walkthrough' -const WALKTHROUGH_VIEWPORT_WIDTH = 1440 -const WALKTHROUGH_VIEWPORT_HEIGHT = 900 -const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg']) - -type RunOptions = { - artifactsDirectory: string - copilotToken: string - dockerImage?: string - maxConcurrency?: number -} - -function getVitestConfig(outputFile: string, browser = false): string { - const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright'\n` : '' - const browserConfig = browser - ? ` browser: { - enabled: true, - headless: true, - provider: playwright(), - instances: [{browser: 'chromium'}], - }, -` - : '' - - return ` -import {defineConfig} from 'vitest/config' -${browserImport} -export default defineConfig({ - test: { -${browserConfig} reporters: [['json', {outputFile: ${JSON.stringify(outputFile)}, includeTaskLocation: true}]], - }, -}) -` -} - -function run(treatments: Array, options: RunOptions): Promise> { - const maxConcurrency = options.maxConcurrency ?? 1 - const queue = treatments.slice() - const results: Array = [] - const pending = new Set() - let cancelled = false - - let resolve: (value: Array) => void - let reject: (reason: unknown) => void - const deferred = new Promise>((_resolve, _reject) => { - resolve = _resolve - reject = _reject - }) - - function execute() { - if (cancelled) { - return - } - - if (queue.length === 0) { - if (pending.size === 0) { - resolve(results) - } - return - } - - if (pending.size >= maxConcurrency) { - return - } - - const treatment = queue.shift() - if (!treatment) { - return - } - - const promise = retry( - () => - runTreatment(treatment, { - artifactsDirectory: options.artifactsDirectory, - copilotToken: options.copilotToken, - dockerImage: options.dockerImage, - }), - 3, - ).then( - result => { - results.push(result) - pending.delete(promise) - execute() - }, - error => { - cancelled = true - pending.delete(promise) - reject(error) - }, - ) - - pending.add(promise) - execute() - } - - execute() - - return deferred -} - -async function retry(fn: () => Promise, retries: number): Promise { - try { - return await fn() - } catch (error) { - if (retries > 0) { - console.log('Retrying after error: %s', error) - return retry(fn, retries - 1) - } - throw error - } -} - -type RunTreatmentOptions = { - artifactsDirectory: string - copilotToken: string - dockerImage?: string -} - -function getCopilotArgs({ - prompt, - model, - reasoningEffort, -}: { - prompt: string - model: Model - reasoningEffort?: ReasoningEffort -}): Array { - const args = ['-p', prompt, '--model', model, '--allow-all'] - - if (reasoningEffort) { - args.push('--reasoning-effort', reasoningEffort) - } - - return [...args, '--mode', 'autopilot', '--output-format', 'json'] -} - -async function runTreatment( - treatment: Treatment, - {artifactsDirectory, copilotToken, dockerImage}: RunTreatmentOptions, -): Promise { - console.log('Running treatment: %s (%s)', treatment.config.name, treatment.id) - await using sandbox = await Sandbox.create({dockerImage}) - - console.log('Copying files from: %s...', treatment.scenario.directory) - await sandbox.copy(treatment.scenario.directory, CONTAINER_WORKDIR, { - exclude: ['scenario.config.ts', 'scenario.test.ts', 'scenario.browser.test.ts', 'node_modules', '.next'], - }) - await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], { - user: 'root', - }) - - console.log('Obfuscating package name...') - await sandbox.runCommand('npm', ['pkg', 'set', `name=${treatment.id}`], { - user: NODE_USER, - }) - - console.log('Removing workspace dependency...') - await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], { - user: NODE_USER, - }) - - console.log('Installing dependencies...') - await sandbox.runCommand('npm', ['install'], { - user: NODE_USER, - }) - - if (treatment.experiment.setup) { - console.log('Running experiment setup...') - await treatment.experiment.setup({ - sandbox, - }) - } - - if (treatment.config.setup) { - console.log('Running treatment setup...') - await treatment.config.setup({ - sandbox, - }) - } - - console.log('Run build script...') - await sandbox.runCommand('npm', ['run', 'build', '--if-present'], { - user: NODE_USER, - }) - - if (treatment.scenario.browserTestPath) { - console.log('Installing browser test dependencies...') - await sandbox.runCommand( - 'npm', - ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'], - { - user: NODE_USER, - }, - ) - console.log('Installing Playwright browser...') - await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], { - user: 'root', - env: { - PLAYWRIGHT_BROWSERS_PATH, - }, - }) - } - - console.log('Running copilot...') - const {prompt} = treatment.scenario.config - const args = getCopilotArgs({ - prompt, - model: treatment.model, - reasoningEffort: treatment.reasoningEffort, - }) - const copilotOutput = await sandbox.runCommand('copilot', args, { - user: NODE_USER, - env: { - COPILOT_GITHUB_TOKEN: copilotToken, - }, - }) - const messages: Array = copilotOutput.stdout.split('\n').flatMap(line => { - const trimmed = line.trim() - if (trimmed.length === 0) { - return [] - } - return parseMessage(JSON.parse(trimmed)) - }) - - const TEST_PATH = 'scenario.test.ts' - const BROWSER_TEST_PATH = 'scenario.browser.test.ts' - const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts' - const TEST_RESULTS_PATH = 'test-results.json' - const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json' - const scenarioTests = [ - { - sourcePath: treatment.scenario.testPath, - testPath: TEST_PATH, - resultsPath: TEST_RESULTS_PATH, - browser: false, - }, - ] - - if (treatment.scenario.browserTestPath) { - scenarioTests.push({ - sourcePath: treatment.scenario.browserTestPath, - testPath: BROWSER_TEST_PATH, - resultsPath: BROWSER_TEST_RESULTS_PATH, - browser: true, - }) - } - - let numFailedTests = 0 - let numPassedTests = 0 - let numPendingTests = 0 - let numTodoTests = 0 - let numTotalTests = 0 - let testRunSuccess = true - const tests: TreatmentResult['testResults']['tests'] = [] - const rawTestResults: Array & {testResults: Array}> = [] - - for (const scenarioTest of scenarioTests) { - await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath) - await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser)) - // Always pass vitest calls even if test suite fails - await sandbox.runCommand( - 'sh', - ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath], - { - user: NODE_USER, - env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : undefined, - }, - ) - - const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath) - const rawTestResult: unknown = JSON.parse(testResultsContent) - const testResults = parseTestResults(rawTestResult) - if (!testResults.success) { - throw new Error(`Failed to parse test results: ${testResults.error}`) - } - - const testSource = await fs.readFile(scenarioTest.sourcePath, 'utf8') - numFailedTests += testResults.data.numFailedTests - numPassedTests += testResults.data.numPassedTests - numPendingTests += testResults.data.numPendingTests - numTodoTests += testResults.data.numTodoTests - numTotalTests += testResults.data.numTotalTests - testRunSuccess &&= testResults.data.success - tests.push(...getTestMetadata(testResults.data, testSource)) - rawTestResults.push(rawTestResult as Record & {testResults: Array}) - } - - if (rawTestResults.length > 1) { - await sandbox.writeFile( - TEST_RESULTS_PATH, - JSON.stringify({ - ...rawTestResults[0], - numFailedTests, - numPassedTests, - numPendingTests, - numTodoTests, - numTotalTests, - success: testRunSuccess, - testResults: rawTestResults.flatMap(testResult => testResult.testResults), - }), - ) - } - - console.log('Capturing walkthrough...') - await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], { - user: 'root', - }) - await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], { - user: NODE_USER, - }) - await sandbox.runCommand( - 'npx', - ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'], - { - user: NODE_USER, - }, - ) - await sandbox.writeFile( - 'agent-browser.json', - JSON.stringify({ - executablePath: CHROMIUM_EXECUTABLE_PATH, - }), - ) - const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves. - -Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything. - -Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project: - -- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png. -- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc. -- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm. - -Only capture the walkthrough, do not make any further code changes.` - const walkthroughResult = await sandbox.runCommand( - 'copilot', - getCopilotArgs({ - prompt: walkthroughPrompt, - model: 'gpt-5.6-terra', - reasoningEffort: 'medium', - }), - { - user: NODE_USER, - env: { - COPILOT_GITHUB_TOKEN: copilotToken, - }, - allowNonZeroExitCode: true, - }, - ) - - if (walkthroughResult.exitCode !== 0) { - console.warn('Unable to capture walkthrough: %s', walkthroughResult.stderr) - } - - // Turns - const assistantTurns = new Set() - // Tools - const toolCalls = new Map() - let outputTokens = 0 - - for (const message of messages) { - if (isMessageType(message, 'assistant.turn_start')) { - assistantTurns.add(message.data.turnId) - } - - if (isMessageType(message, 'assistant.message')) { - outputTokens += message.data.outputTokens ?? 0 - } - - if (isMessageType(message, 'tool.execution_start')) { - const toolName = message.data.toolName - toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1) - } - } - - const result = messages.find(message => isMessageType(message, 'result')) - if (!result) { - throw new Error('No result message found in copilot output') - } - - const artifactDirectory = path.join(artifactsDirectory, treatment.id) - const workspacePath = path.join(artifactDirectory, 'workspace') - const walkthroughPath = path.join(artifactDirectory, 'walkthrough') - const copilotConfigPath = path.join(artifactDirectory, '.copilot') - const skillsConfigPath = path.join(artifactDirectory, '.agents') - const testResultsPath = path.join(workspacePath, 'test-results.json') - await fs.rm(artifactDirectory, {recursive: true, force: true}) - await fs.mkdir(workspacePath, {recursive: true}) - - console.log('Downloading agent workspace to: %s...', workspacePath) - await sandbox.download(CONTAINER_WORKDIR, workspacePath, { - ignore(name) { - return name.includes('node_modules') || name.includes('.next') || name.includes('dist') - }, - }) - - console.log('Downloading copilot config to: %s...', copilotConfigPath) - await sandbox.download(COPILOT_DIR, copilotConfigPath) - - console.log('Downloading skills config to: %s...', skillsConfigPath) - await sandbox.download(AGENTS_DIR, skillsConfigPath) - - let walkthrough: Walkthrough = { - type: 'Unavailable', - } - - if (existsSync(path.join(workspacePath, WALKTHROUGH_DIR))) { - console.log( - 'Moving walkthrough artifacts from: %s to: %s...', - path.join(workspacePath, WALKTHROUGH_DIR), - walkthroughPath, - ) - await fs.mkdir(walkthroughPath, {recursive: true}) - await fs.rename(path.join(workspacePath, WALKTHROUGH_DIR), walkthroughPath) - - if (existsSync(path.join(walkthroughPath, 'screenshot.png'))) { - walkthrough = { - type: 'Screenshot', - filepath: path.join(walkthroughPath, 'screenshot.png'), - } - } else if (existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) { - walkthrough = { - type: 'Video', - filepath: path.join(walkthroughPath, 'walkthrough.webm'), - } - } else if (existsSync(path.join(walkthroughPath, 'screenshots'))) { - const screenshotsDir = path.join(walkthroughPath, 'screenshots') - const entries = await fs.readdir(screenshotsDir).then(filenames => { - return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true})) - }) - const screenshots = entries.filter(entry => { - return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase()) - }) - if (screenshots.length > 0) { - walkthrough = { - type: 'Screenshots', - screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)), - } - } - } - } - - return { - id: randomUUID(), - treatment, - artifacts: { - directory: artifactDirectory, - copilotConfigPath, - skillsConfigPath, - testResultsPath, - workspacePath, - }, - assistant: { - logs: messages, - turns: assistantTurns.size, - outputTokens, - premiumRequests: result.usage.premiumRequests, - // Time to complete (latency) - totalApiDurationMs: result.usage.totalApiDurationMs, - sessionDurationMs: result.usage.sessionDurationMs, - tools: Object.fromEntries(toolCalls), - }, - testResults: { - numFailedTests, - numPassedTests, - numPendingTests, - numTodoTests, - numTotalTests, - tests, - }, - walkthrough, - } -} - -export {getCopilotArgs, getVitestConfig, run} diff --git a/packages/agent-eval/src/sandbox.test.ts b/packages/agent-eval/src/sandbox.test.ts deleted file mode 100644 index e468228c..00000000 --- a/packages/agent-eval/src/sandbox.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import {describe, expect, test, vi} from 'vitest' -import {Sandbox} from './sandbox' - -function createSandbox() { - const sandbox = new Sandbox({} as never, {} as never) - const copy = vi.spyOn(sandbox, 'copy').mockResolvedValue() - const runCommand = vi.spyOn(sandbox, 'runCommand').mockResolvedValue({ - stdout: '', - stderr: '', - exitCode: 0, - }) - - return {sandbox, copy, runCommand} -} - -describe('addCopilotPlugin', () => { - test('installs a remote plugin', async () => { - const {sandbox, copy, runCommand} = createSandbox() - - await sandbox.addCopilotPlugin({ - type: 'remote', - url: 'https://github.com/example/plugin.git', - }) - - expect(copy).not.toHaveBeenCalled() - expect(runCommand).toHaveBeenCalledOnce() - expect(runCommand).toHaveBeenCalledWith('copilot', ['plugin', 'install', 'https://github.com/example/plugin.git']) - }) - - test('installs a versioned remote plugin', async () => { - const {sandbox, copy, runCommand} = createSandbox() - - await sandbox.addCopilotPlugin({ - type: 'remote', - url: 'https://github.com/example/plugin.git', - version: 'v1.2.3', - }) - - expect(copy).not.toHaveBeenCalled() - expect(runCommand).toHaveBeenCalledOnce() - expect(runCommand).toHaveBeenCalledWith('copilot', [ - 'plugin', - 'install', - 'https://github.com/example/plugin.git#v1.2.3', - ]) - }) - - test('copies and installs a local plugin', async () => { - const {sandbox, copy, runCommand} = createSandbox() - - await sandbox.addCopilotPlugin({ - type: 'local', - sourcePath: './plugins/local-plugin', - }) - - expect(copy).toHaveBeenCalledOnce() - const pluginPath = copy.mock.calls[0][1] - expect(pluginPath).toMatch(/^\/home\/node\/\.copilot\/plugin-sources\//) - expect(copy).toHaveBeenCalledWith('./plugins/local-plugin', pluginPath) - expect(runCommand).toHaveBeenCalledOnce() - expect(runCommand).toHaveBeenCalledWith('copilot', ['plugin', 'install', pluginPath]) - }) - - test('adds a remote marketplace and installs its plugin', async () => { - const {sandbox, copy, runCommand} = createSandbox() - - await sandbox.addCopilotPlugin({ - type: 'marketplace', - name: 'example-plugin', - marketplace: { - name: 'example-marketplace', - source: { - type: 'remote', - url: 'https://github.com/example/marketplace.git', - version: 'v1.2.3', - }, - }, - }) - - expect(copy).not.toHaveBeenCalled() - expect(runCommand).toHaveBeenNthCalledWith(1, 'copilot', [ - 'plugin', - 'marketplace', - 'add', - 'https://github.com/example/marketplace.git#v1.2.3', - ]) - expect(runCommand).toHaveBeenNthCalledWith(2, 'copilot', [ - 'plugin', - 'install', - 'example-plugin@example-marketplace', - ]) - }) - - test('copies a local marketplace and installs its plugin', async () => { - const {sandbox, copy, runCommand} = createSandbox() - - await sandbox.addCopilotPlugin({ - type: 'marketplace', - name: 'example-plugin', - marketplace: { - name: 'example-marketplace', - source: { - type: 'local', - sourcePath: './plugins/marketplace', - }, - }, - }) - - const marketplacePath = copy.mock.calls[0][1] - expect(marketplacePath).toMatch(/^\/home\/node\/\.copilot\/plugin-sources\//) - expect(copy).toHaveBeenCalledWith('./plugins/marketplace', marketplacePath) - expect(runCommand).toHaveBeenNthCalledWith(1, 'copilot', ['plugin', 'marketplace', 'add', marketplacePath]) - expect(runCommand).toHaveBeenNthCalledWith(2, 'copilot', [ - 'plugin', - 'install', - 'example-plugin@example-marketplace', - ]) - }) -}) diff --git a/packages/agent-eval/src/sandbox/captured-stream.test.ts b/packages/agent-eval/src/sandbox/captured-stream.test.ts new file mode 100644 index 00000000..20f64d3f --- /dev/null +++ b/packages/agent-eval/src/sandbox/captured-stream.test.ts @@ -0,0 +1,50 @@ +import {describe, expect, test, vi} from 'vitest' +import {createCapturedStream} from './captured-stream' + +describe('createCapturedStream', () => { + test('logs complete lines while preserving captured output', () => { + const onLine = vi.fn() + const captured = createCapturedStream(onLine) + + captured.stream.write('first line\nsecond ') + captured.stream.write('line\r\nthird line\r') + captured.stream.write('\nfourth line\rfifth line') + captured.flush() + + expect(onLine.mock.calls).toEqual([ + ['first line'], + ['second line'], + ['third line'], + ['fourth line'], + ['fifth line'], + ]) + expect(captured.read()).toBe('first line\nsecond line\r\nthird line\r\nfourth line\rfifth line') + }) + + test('flushes a final line without a delimiter', () => { + const onLine = vi.fn() + const captured = createCapturedStream(onLine) + + captured.stream.write('final line') + + expect(onLine).not.toHaveBeenCalled() + + captured.flush() + + expect(onLine).toHaveBeenCalledOnce() + expect(onLine).toHaveBeenCalledWith('final line') + }) + + test('preserves multibyte characters split across chunks', () => { + const onLine = vi.fn() + const captured = createCapturedStream(onLine) + const output = Buffer.from('hello 👋\n') + + captured.stream.write(output.subarray(0, output.length - 3)) + captured.stream.write(output.subarray(output.length - 3)) + captured.flush() + + expect(onLine).toHaveBeenCalledWith('hello 👋') + expect(captured.read()).toBe('hello 👋\n') + }) +}) diff --git a/packages/agent-eval/src/sandbox/captured-stream.ts b/packages/agent-eval/src/sandbox/captured-stream.ts new file mode 100644 index 00000000..7c14aa1f --- /dev/null +++ b/packages/agent-eval/src/sandbox/captured-stream.ts @@ -0,0 +1,59 @@ +import {StringDecoder} from 'node:string_decoder' +import {Writable} from 'node:stream' + +type CapturedStream = { + stream: Writable + read(): string + flush(): void +} + +function createCapturedStream(onLine: (line: string) => void): CapturedStream { + const chunks: Array = [] + const decoder = new StringDecoder('utf8') + let pending = '' + + function emitLines(final: boolean): void { + while (pending.length > 0) { + const delimiterIndex = pending.search(/[\r\n]/) + if (delimiterIndex === -1) { + break + } + + if (pending[delimiterIndex] === '\r' && delimiterIndex === pending.length - 1 && !final) { + break + } + + const delimiterLength = pending.slice(delimiterIndex, delimiterIndex + 2) === '\r\n' ? 2 : 1 + onLine(pending.slice(0, delimiterIndex)) + pending = pending.slice(delimiterIndex + delimiterLength) + } + + if (final && pending.length > 0) { + onLine(pending) + pending = '' + } + } + + const stream = new Writable({ + write(chunk: Buffer | string, encoding, callback) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding) + chunks.push(buffer) + pending += decoder.write(buffer) + emitLines(false) + callback() + }, + }) + + return { + stream, + read() { + return Buffer.concat(chunks).toString('utf8') + }, + flush() { + pending += decoder.end() + emitLines(true) + }, + } +} + +export {createCapturedStream} diff --git a/packages/agent-eval/src/sandbox/constants.test.ts b/packages/agent-eval/src/sandbox/constants.test.ts new file mode 100644 index 00000000..17c9943d --- /dev/null +++ b/packages/agent-eval/src/sandbox/constants.test.ts @@ -0,0 +1,26 @@ +import path from 'node:path' +import {expect, test} from 'vitest' +import { + AGENT_INSTRUCTIONS_PATH, + AGENTS_DIR, + CONTAINER_WORKDIR, + COPILOT_DIR, + COPILOT_PLUGIN_SOURCES_DIR, + CUSTOM_AGENTS_DIR, + MCP_CONFIG_PATH, + NODE_USER, + NPM_GLOBAL_DIR, + SANDBOX_GID, + SANDBOX_UID, + SKILLS_DIR, +} from './constants' + +test('sandbox paths and user values remain internally consistent', () => { + expect(AGENT_INSTRUCTIONS_PATH).toBe(path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md')) + expect(CUSTOM_AGENTS_DIR).toBe(path.posix.join(COPILOT_DIR, 'agents')) + expect(COPILOT_PLUGIN_SOURCES_DIR).toBe(path.posix.join(COPILOT_DIR, 'plugin-sources')) + expect(MCP_CONFIG_PATH).toBe(path.posix.join(COPILOT_DIR, 'mcp-config.json')) + expect(SKILLS_DIR).toBe(path.posix.join(AGENTS_DIR, 'skills')) + expect(NODE_USER).toBe(`${SANDBOX_UID}:${SANDBOX_GID}`) + expect(NPM_GLOBAL_DIR).toBe('/home/node/.npm-global') +}) diff --git a/packages/agent-eval/src/sandbox/constants.ts b/packages/agent-eval/src/sandbox/constants.ts new file mode 100644 index 00000000..e9b4d48a --- /dev/null +++ b/packages/agent-eval/src/sandbox/constants.ts @@ -0,0 +1,68 @@ +import path from 'node:path' + +/** + * Working directory inside the sandbox. + */ +const CONTAINER_WORKDIR = '/home/sandbox/workspace' + +/** + * Directory for Copilot CLI configuration. + */ +const COPILOT_DIR = '/home/node/.copilot' + +/** + * Directory for custom Copilot agents. + */ +const CUSTOM_AGENTS_DIR = '/home/node/.copilot/agents' + +/** + * Directory for agent configuration and skills. + */ +const AGENTS_DIR = '/home/node/.agents' + +/** + * Directory for agent skills. + */ +const SKILLS_DIR = '/home/node/.agents/skills' + +/** + * Directory for local plugin sources copied into the sandbox. + */ +const COPILOT_PLUGIN_SOURCES_DIR = path.posix.join(COPILOT_DIR, 'plugin-sources') + +/** + * Path for project agent instructions. + */ +const AGENT_INSTRUCTIONS_PATH = path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md') + +/** + * Path for MCP server configuration. + */ +const MCP_CONFIG_PATH = path.posix.join(COPILOT_DIR, 'mcp-config.json') + +/** + * Node.js images provide a non-root node user with this UID and GID. + */ +const SANDBOX_UID = 1000 +const SANDBOX_GID = 1000 +const NODE_USER = `${SANDBOX_UID}:${SANDBOX_GID}` as const + +/** + * Directory for npm packages installed globally by the non-root user. + */ +const NPM_GLOBAL_DIR = '/home/node/.npm-global' + +export { + AGENT_INSTRUCTIONS_PATH, + AGENTS_DIR, + CONTAINER_WORKDIR, + COPILOT_DIR, + COPILOT_PLUGIN_SOURCES_DIR, + CUSTOM_AGENTS_DIR, + MCP_CONFIG_PATH, + NODE_USER, + NPM_GLOBAL_DIR, + SANDBOX_GID, + SANDBOX_UID, + SKILLS_DIR, +} diff --git a/packages/agent-eval/src/sandbox/index.test.ts b/packages/agent-eval/src/sandbox/index.test.ts new file mode 100644 index 00000000..8f1cff0e --- /dev/null +++ b/packages/agent-eval/src/sandbox/index.test.ts @@ -0,0 +1,10 @@ +import {expect, test} from 'vitest' +import {CONTAINER_WORKDIR, DEFAULT_DOCKER_IMAGE, SandboxSchema, SystemSandbox, VirtualSandbox} from './index' + +test('exports sandbox implementations, schema, and constants', () => { + expect(CONTAINER_WORKDIR).toBe('/home/sandbox/workspace') + expect(DEFAULT_DOCKER_IMAGE).toBe('node:26.5.0-slim') + expect(SandboxSchema).toBeDefined() + expect(SystemSandbox).toBeTypeOf('function') + expect(VirtualSandbox).toBeTypeOf('function') +}) diff --git a/packages/agent-eval/src/sandbox/index.ts b/packages/agent-eval/src/sandbox/index.ts new file mode 100644 index 00000000..c49da6f4 --- /dev/null +++ b/packages/agent-eval/src/sandbox/index.ts @@ -0,0 +1,25 @@ +export * from './constants' +export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE} from './system' +export {VirtualSandbox} from './virtual' +export type { + AgentSkillCopiedFile, + AgentSkillFile, + AgentSkillOptions, + AgentSkillWrittenFile, + CommandResult, + CopilotPluginConfig, + CopilotPluginSource, + CopyOptions, + CustomAgentCopiedFile, + CustomAgentFile, + CustomAgentOptions, + CustomAgentWrittenFile, + DownloadOptions, + LocalCopilotPluginSource, + McpServerConfig, + RemoteCopilotPluginSource, + RunOptions, + Sandbox, + SandboxConstructor, + SandboxCreateOptions, +} from './types' diff --git a/packages/agent-eval/src/sandbox/path.test.ts b/packages/agent-eval/src/sandbox/path.test.ts new file mode 100644 index 00000000..a78a74de --- /dev/null +++ b/packages/agent-eval/src/sandbox/path.test.ts @@ -0,0 +1,13 @@ +import {describe, expect, test} from 'vitest' +import {CONTAINER_WORKDIR} from './constants' +import {resolveContainerPath} from './path' + +describe('resolveContainerPath', () => { + test('resolves relative paths from the container workdir', () => { + expect(resolveContainerPath('nested/../example.txt')).toBe(`${CONTAINER_WORKDIR}/example.txt`) + }) + + test('normalizes absolute container paths', () => { + expect(resolveContainerPath('/tmp/nested/../example.txt')).toBe('/tmp/example.txt') + }) +}) diff --git a/packages/agent-eval/src/sandbox/path.ts b/packages/agent-eval/src/sandbox/path.ts new file mode 100644 index 00000000..febc7e28 --- /dev/null +++ b/packages/agent-eval/src/sandbox/path.ts @@ -0,0 +1,12 @@ +import path from 'node:path' +import {CONTAINER_WORKDIR} from './constants' + +function resolveContainerPath(filepath: string): string { + if (path.posix.isAbsolute(filepath)) { + return path.posix.normalize(filepath) + } + + return path.posix.resolve(CONTAINER_WORKDIR, filepath) +} + +export {resolveContainerPath} diff --git a/packages/agent-eval/src/sandbox/system.test.ts b/packages/agent-eval/src/sandbox/system.test.ts new file mode 100644 index 00000000..238ed33c --- /dev/null +++ b/packages/agent-eval/src/sandbox/system.test.ts @@ -0,0 +1,260 @@ +import Docker from 'dockerode' +import {beforeEach, describe, expect, test, vi} from 'vitest' +import {VirtualHost} from '../host' +import {MCP_CONFIG_PATH, NODE_USER, SKILLS_DIR} from './constants' +import {createContainer, SandboxSchema, SystemSandbox} from './system' +import {VirtualSandbox} from './virtual' + +function createSandbox(container = {remove: vi.fn()}) { + // @ts-expect-error This test only exercises methods whose container operations are mocked. + return new SystemSandbox(VirtualHost.create(), new Docker(), container) +} + +describe('SandboxSchema', () => { + test('accepts system and virtual sandboxes', async () => { + const systemSandbox = createSandbox() + const virtualSandbox = await VirtualSandbox.create() + + expect(SandboxSchema.parse(systemSandbox)).toBe(systemSandbox) + expect(SandboxSchema.parse(virtualSandbox)).toBe(virtualSandbox) + expect(() => { + SandboxSchema.parse({}) + }).toThrow() + }) +}) + +describe('SystemSandbox lifecycle', () => { + test('force removes the container when disposed', async () => { + const container = { + remove: vi.fn(), + } + const sandbox = createSandbox(container) + + await sandbox[Symbol.asyncDispose]() + + expect(container.remove).toHaveBeenCalledWith({force: true}) + }) + + test('force removes the container when initialization fails', async () => { + const initializationError = new Error('Failed to start container') + const container = { + start: vi.fn().mockRejectedValue(initializationError), + remove: vi.fn(), + } + const docker = { + createContainer: vi.fn().mockResolvedValue(container), + pull: vi.fn((_name: string, callback: (error: Error | null, stream: NodeJS.ReadableStream) => void) => { + callback(null, {} as NodeJS.ReadableStream) + }), + modem: { + followProgress: vi.fn((_stream: NodeJS.ReadableStream, onFinished: (error: Error | null) => void) => { + onFinished(null) + }), + }, + } + + // @ts-expect-error This test only exercises the Docker methods used before container initialization. + await expect(createContainer(docker, 'test-image')).rejects.toBe(initializationError) + expect(container.remove).toHaveBeenCalledWith({force: true}) + }) +}) + +describe('SystemSandbox configuration helpers', () => { + let sandbox: SystemSandbox + + beforeEach(() => { + sandbox = createSandbox() + vi.spyOn(sandbox, 'runCommand').mockResolvedValue({ + stdout: '', + stderr: '', + exitCode: 0, + }) + vi.spyOn(sandbox, 'writeFile').mockResolvedValue() + vi.spyOn(sandbox, 'copy').mockResolvedValue() + }) + + test('appends agent instructions with normalized newlines', async () => { + vi.spyOn(sandbox, 'exists').mockResolvedValue(true) + vi.spyOn(sandbox, 'readFile').mockResolvedValue('Existing instructions') + + await sandbox.addAgentInstruction('New instructions') + + expect(sandbox.writeFile).toHaveBeenCalledWith( + '/home/sandbox/workspace/AGENTS.md', + 'Existing instructions\nNew instructions\n', + ) + }) + + test('creates an agent skill and supporting files', async () => { + vi.spyOn(sandbox, 'exists').mockResolvedValue(false) + + await sandbox.addAgentSkill('example-skill', 'Example description', 'Skill instructions', { + files: [ + { + path: 'references/example.md', + content: 'reference', + }, + { + sourcePath: '/fixtures/script.js', + destinationPath: 'scripts/script.js', + }, + ], + }) + + expect(sandbox.writeFile).toHaveBeenCalledWith( + `${SKILLS_DIR}/example-skill/SKILL.md`, + `--- +name: "example-skill" +description: "Example description" +--- + +Skill instructions +`, + ) + expect(sandbox.writeFile).toHaveBeenCalledWith(`${SKILLS_DIR}/example-skill/references/example.md`, 'reference') + expect(sandbox.copy).toHaveBeenCalledWith('/fixtures/script.js', `${SKILLS_DIR}/example-skill/scripts/script.js`) + }) + + test('rejects invalid skill names and file destinations', async () => { + vi.spyOn(sandbox, 'exists').mockResolvedValue(false) + + await expect(sandbox.addAgentSkill('Invalid Skill', 'description', 'contents')).rejects.toThrow( + 'Skill names must be lowercase and use hyphens for spaces', + ) + await expect( + sandbox.addAgentSkill('valid-skill', 'description', 'contents', { + files: [ + { + path: '../outside.md', + content: 'outside', + }, + ], + }), + ).rejects.toThrow('Invalid agent skill file destination "../outside.md"') + }) + + test('creates a custom agent with tools', async () => { + vi.spyOn(sandbox, 'exists').mockResolvedValue(false) + + await sandbox.addCustomAgent('example-agent', 'Example description', 'Agent instructions', { + tools: ['view', 'grep'], + }) + + expect(sandbox.writeFile).toHaveBeenCalledWith( + '/home/node/.copilot/agents/example-agent.agent.md', + `--- +name: "example-agent" +description: "Example description" +tools: ["view","grep"] +--- + +Agent instructions +`, + ) + }) + + test('adds an MCP server to the existing configuration', async () => { + vi.spyOn(sandbox, 'readFile').mockResolvedValue( + JSON.stringify({ + mcpServers: { + existing: { + command: 'existing-server', + type: 'local', + }, + }, + }), + ) + + await sandbox.addMcpServer('example', { + command: 'example-server', + type: 'local', + }) + + expect(sandbox.writeFile).toHaveBeenCalledWith( + MCP_CONFIG_PATH, + JSON.stringify( + { + mcpServers: { + existing: { + command: 'existing-server', + type: 'local', + }, + example: { + command: 'example-server', + type: 'local', + }, + }, + }, + null, + 2, + ), + ) + expect(sandbox.runCommand).toHaveBeenCalledWith('chown', ['-R', NODE_USER, MCP_CONFIG_PATH], { + user: 'root', + }) + }) + + test('rejects duplicate MCP server names', async () => { + vi.spyOn(sandbox, 'readFile').mockResolvedValue( + JSON.stringify({ + mcpServers: { + example: { + command: 'example-server', + type: 'local', + }, + }, + }), + ) + + await expect( + sandbox.addMcpServer('example', { + command: 'other-server', + type: 'local', + }), + ).rejects.toThrow('MCP server with name "example" already exists') + }) + + test('installs remote, local, and marketplace plugins', async () => { + await sandbox.addCopilotPlugin({ + type: 'remote', + url: 'https://example.com/plugin.git', + version: 'v1', + }) + await sandbox.addCopilotPlugin({ + type: 'local', + sourcePath: '/fixtures/plugin', + }) + await sandbox.addCopilotPlugin({ + type: 'marketplace', + name: 'example-plugin', + marketplace: { + name: 'example-marketplace', + source: { + type: 'remote', + url: 'https://example.com/marketplace.git', + }, + }, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [ + 'plugin', + 'install', + 'https://example.com/plugin.git#v1', + ]) + expect(sandbox.copy).toHaveBeenCalledWith( + '/fixtures/plugin', + expect.stringMatching(/^\/home\/node\/\.copilot\/plugin-sources\/.+/), + ) + expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [ + 'plugin', + 'marketplace', + 'add', + 'https://example.com/marketplace.git', + ]) + expect(sandbox.runCommand).toHaveBeenCalledWith('copilot', [ + 'plugin', + 'install', + 'example-plugin@example-marketplace', + ]) + }) +}) diff --git a/packages/agent-eval/src/sandbox.ts b/packages/agent-eval/src/sandbox/system.ts similarity index 70% rename from packages/agent-eval/src/sandbox.ts rename to packages/agent-eval/src/sandbox/system.ts index b733aab0..6db12ba4 100644 --- a/packages/agent-eval/src/sandbox.ts +++ b/packages/agent-eval/src/sandbox/system.ts @@ -1,169 +1,82 @@ import {randomUUID} from 'node:crypto' -import fs from 'node:fs/promises' import path from 'node:path' -import {Writable} from 'node:stream' import {pipeline} from 'node:stream/promises' import Docker from 'dockerode' import tarFs from 'tar-fs' import type {Headers} from 'tar-fs' import tarStream from 'tar-stream' -import {McpConfigFileSchema} from './mcp-config' -import type {McpConfigFile, McpServerConfig} from './mcp-config' - -const COPILOT_CLI_VERSION = '1.0.80' +import * as z from 'zod/mini' +import {McpConfigFileSchema} from '../mcp-config' +import type {McpConfigFile} from '../mcp-config' +import { + AGENT_INSTRUCTIONS_PATH, + AGENTS_DIR, + CONTAINER_WORKDIR, + COPILOT_DIR, + COPILOT_PLUGIN_SOURCES_DIR, + CUSTOM_AGENTS_DIR, + MCP_CONFIG_PATH, + NODE_USER, + NPM_GLOBAL_DIR, + SANDBOX_GID, + SANDBOX_UID, + SKILLS_DIR, +} from './constants' +import type { + AgentSkillFile, + AgentSkillOptions, + AgentSkillWrittenFile, + CommandResult, + CopilotPluginConfig, + CopilotPluginSource, + CopyOptions, + CustomAgentFile, + CustomAgentOptions, + CustomAgentWrittenFile, + DownloadOptions, + McpServerConfig, + RunOptions, + Sandbox, + SandboxCreateOptions, +} from './types' +import {DefaultHost, type Host} from '../host' +import {VirtualSandbox} from './virtual' +import {resolveContainerPath} from './path' +import {logger} from '../logger' +import {createCapturedStream} from './captured-stream' + +const COPILOT_CLI_VERSION = '1.0.82' const NPM_VERSION = '12.0.2' -/** - * Working directory inside the container. - */ -const CONTAINER_WORKDIR = '/home/sandbox/workspace' - -/** - * Directory for copilot cli configuration. - */ -const COPILOT_DIR = '/home/node/.copilot' - -/** - * Directory for custom Copilot sub-agents. - */ -const CUSTOM_AGENTS_DIR = '/home/node/.copilot/agents' - -/** - * Directory for agents configuration and skills. - */ -const AGENTS_DIR = '/home/node/.agents' - -/** - * Directory for skills. - */ -const SKILLS_DIR = '/home/node/.agents/skills' - -/** - * Directory for local plugin sources copied into the container. - */ -const COPILOT_PLUGIN_SOURCES_DIR = path.posix.join(COPILOT_DIR, 'plugin-sources') - -/** - * Path for project agent instructions. - */ -const AGENT_INSTRUCTIONS_PATH = path.posix.join(CONTAINER_WORKDIR, 'AGENTS.md') - -/** - * Path for MCP server configuration file. - */ -const MCP_CONFIG_PATH = path.join(COPILOT_DIR, 'mcp-config.json') - -/** - * Non-root user configuration. - * Running as non-root is important for security and compatibility - * (e.g., Claude Code refuses --dangerously-skip-permissions as root). - * Node.js images already have a 'node' user with UID/GID 1000. - */ -const SANDBOX_UID = 1000 -const SANDBOX_GID = 1000 -const NODE_USER = `${SANDBOX_UID}:${SANDBOX_GID}` as const - -/** - * Directory for npm global packages (non-root install location). - */ -const NPM_GLOBAL_DIR = '/home/node/.npm-global' - -type RunOptions = { - env?: Record - user?: string - allowNonZeroExitCode?: boolean -} - -type CopyOptions = { - exclude?: string[] -} - -type CustomAgentCopiedFile = { - sourcePath: string - destinationPath?: string -} - -type CustomAgentWrittenFile = { - path: string - content: string -} - -type CustomAgentFile = CustomAgentCopiedFile | CustomAgentWrittenFile - -type AgentSkillCopiedFile = CustomAgentCopiedFile - -type AgentSkillWrittenFile = CustomAgentWrittenFile - -type AgentSkillFile = AgentSkillCopiedFile | AgentSkillWrittenFile - -type AgentSkillOptions = { - files?: Array -} - -type RemoteCopilotPluginSource = { - type: 'remote' - url: string - version?: string -} - -type LocalCopilotPluginSource = { - type: 'local' - sourcePath: string -} - -type CopilotPluginSource = RemoteCopilotPluginSource | LocalCopilotPluginSource - -type CopilotPluginConfig = - | CopilotPluginSource - | { - type: 'marketplace' - name: string - marketplace: { - name: string - source: CopilotPluginSource - } - } - -type CustomAgentOptions = { - files?: Array - tools?: Array -} - -type DownloadOptions = { - ignore?: (name: string) => boolean -} - -type SandboxCreateOptions = { - dockerImage?: string -} - const DEFAULT_MCP_CONFIG: McpConfigFile = { mcpServers: {}, } -class Sandbox { +class SystemSandbox implements Sandbox { static async create(options: SandboxCreateOptions = {}) { const docker = new Docker() const dockerImage = options.dockerImage?.trim() || DEFAULT_DOCKER_IMAGE const container = await createContainer(docker, dockerImage) - return new Sandbox(docker, container) + return new SystemSandbox(options.host ?? DefaultHost, docker, container) } - #docker: Docker #container: Docker.Container + #docker: Docker + #host: Host - constructor(docker: Docker, container: InitializedContainer) { + constructor(host: Host, docker: Docker, container: InitializedContainer) { + this.#host = host this.#docker = docker this.#container = container } async [Symbol.asyncDispose]() { - await this.#container.stop() + await this.#container.remove({force: true}) } async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise { const source = path.resolve(sourcePath) - const sourceStats = await fs.stat(source) + const sourceStats = await this.#host.fs.stat(source) if (!sourceStats.isDirectory() && !sourceStats.isFile()) { throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`) } @@ -200,14 +113,16 @@ class Sandbox { } async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise { - await fs.mkdir(hostDestinationPath, { + const containerPath = resolveContainerPath(containerFilePath) + + await this.#host.fs.mkdir(hostDestinationPath, { recursive: true, }) const archive = await this.#container.getArchive({ - path: containerFilePath, + path: containerPath, }) - const sourceName = path.posix.basename(containerFilePath) + const sourceName = path.posix.basename(containerPath) await pipeline( archive, @@ -272,6 +187,7 @@ class Sandbox { } async runCommand(command: string, args: Array = [], options?: RunOptions): Promise { + logger.debug('[sandbox] Running command: %s %s', command, args.join(' ')) return execCommand(this.#docker, this.#container, command, args, { env: { HOME: options?.user === 'root' ? '/root' : '/home/node', @@ -419,89 +335,101 @@ async function createContainer(docker: Docker, dockerImage: string): Promise ${MCP_CONFIG_PATH}`], + { + user: NODE_USER, + }, + ) + await execCommand(docker, container, 'mkdir', ['-p', CUSTOM_AGENTS_DIR], { + user: NODE_USER, + }) - return container as InitializedContainer -} + logger.debug('Setting up agents config...') + await execCommand(docker, container, 'mkdir', ['-p', AGENTS_DIR], { + user: 'root', + }) + await execCommand(docker, container, 'chown', ['-R', NODE_USER, AGENTS_DIR], { + user: 'root', + }) -function resolveContainerPath(filepath: string): string { - if (path.posix.isAbsolute(filepath)) { - return filepath + return container as InitializedContainer + } catch (error) { + try { + await container.remove({force: true}) + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], 'Failed to initialize and remove sandbox container', { + cause: cleanupError, + }) + } + throw error } - - return path.posix.join(CONTAINER_WORKDIR, filepath) } function mapCopiedHeader(header: Headers, sourceName: string, destinationName: string): Headers { @@ -688,12 +616,6 @@ function pullImage(docker: Docker, name: string): Promise { }) } -type CommandResult = { - stdout: string - stderr: string - exitCode: number -} - class CommandError extends Error { command: ReadonlyArray result: CommandResult @@ -730,13 +652,20 @@ async function execCommand( }) return new Promise((resolve, reject) => { - const stdout = captureStream(process.stdout) - const stderr = captureStream(process.stderr) + const stdout = createCapturedStream(line => { + logger.debug('[sandbox]: %s', line) + }) + const stderr = createCapturedStream(line => { + logger.debug('[sandbox]: %s', line) + }) docker.modem.demuxStream(stream, stdout.stream, stderr.stream) stream.on('end', async () => { try { + stdout.flush() + stderr.flush() + const inspectInfo = await exec.inspect() const exitCode = inspectInfo.ExitCode ?? 0 const result = { @@ -755,42 +684,16 @@ async function execCommand( reject(error) } }) - stream.on('error', reject) + stream.on('error', error => { + stdout.flush() + stderr.flush() + reject(error) + }) }) } -function captureStream(destination: NodeJS.WritableStream): {stream: Writable; read(): string} { - const chunks: Array = [] - const stream = new Writable({ - write(chunk: Buffer | string, encoding, callback) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, encoding) - chunks.push(buffer) - destination.write(buffer) - callback() - }, - }) - - return { - stream, - read() { - return Buffer.concat(chunks).toString('utf8') - }, - } -} +const SandboxSchema = z.custom(value => { + return value instanceof SystemSandbox || value instanceof VirtualSandbox +}) -export {CONTAINER_WORKDIR, COPILOT_DIR, CUSTOM_AGENTS_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox} -export type { - AgentSkillCopiedFile, - AgentSkillFile, - AgentSkillOptions, - AgentSkillWrittenFile, - CopilotPluginConfig, - CopilotPluginSource, - CustomAgentCopiedFile, - CustomAgentFile, - CustomAgentOptions, - CustomAgentWrittenFile, - LocalCopilotPluginSource, - RemoteCopilotPluginSource, -} -export type {McpServerConfig} from './mcp-config' +export {SandboxSchema, SystemSandbox, DEFAULT_DOCKER_IMAGE, createContainer} diff --git a/packages/agent-eval/src/sandbox/types.ts b/packages/agent-eval/src/sandbox/types.ts new file mode 100644 index 00000000..6fb23d10 --- /dev/null +++ b/packages/agent-eval/src/sandbox/types.ts @@ -0,0 +1,170 @@ +import type {Host} from '../host' +import type {McpServerConfig} from '../mcp-config' + +type RunOptions = { + env?: Record + user?: string + allowNonZeroExitCode?: boolean +} + +type CopyOptions = { + exclude?: Array +} + +type DownloadOptions = { + ignore?: (name: string) => boolean +} + +type CommandResult = { + stdout: string + stderr: string + exitCode: number +} + +type SandboxCreateOptions = { + dockerImage?: string + host?: Host +} + +type CustomAgentCopiedFile = { + sourcePath: string + destinationPath?: string +} + +type CustomAgentWrittenFile = { + path: string + content: string +} + +type CustomAgentFile = CustomAgentCopiedFile | CustomAgentWrittenFile + +type AgentSkillCopiedFile = CustomAgentCopiedFile + +type AgentSkillWrittenFile = CustomAgentWrittenFile + +type AgentSkillFile = AgentSkillCopiedFile | AgentSkillWrittenFile + +type AgentSkillOptions = { + files?: Array +} + +type RemoteCopilotPluginSource = { + type: 'remote' + url: string + version?: string +} + +type LocalCopilotPluginSource = { + type: 'local' + sourcePath: string +} + +type CopilotPluginSource = RemoteCopilotPluginSource | LocalCopilotPluginSource + +type CopilotPluginConfig = + | CopilotPluginSource + | { + type: 'marketplace' + name: string + marketplace: { + name: string + source: CopilotPluginSource + } + } + +type CustomAgentOptions = { + files?: Array + tools?: Array +} + +interface Sandbox { + /** + * Stops and removes the sandbox container. + */ + [Symbol.asyncDispose](): Promise + + /** + * Copies a host file or directory into the sandbox. + */ + copy(sourcePath: string, destinationPath: string, options?: CopyOptions): Promise + + /** + * Downloads a file or directory from the sandbox to the host. + */ + download(containerFilePath: string, hostDestinationPath: string, options?: DownloadOptions): Promise + + /** + * Reads a UTF-8 file from the sandbox. + */ + readFile(filepath: string): Promise + + /** + * Writes a UTF-8 file to the sandbox. + */ + writeFile(filepath: string, contents: string): Promise + + /** + * Checks whether a file or directory exists in the sandbox. + */ + exists(filepath: string): Promise + + /** + * Runs a command in the sandbox and captures its output and exit code. + */ + runCommand(command: string, args?: Array, options?: RunOptions): Promise + + /** + * Appends instructions to the sandbox's project-level AGENTS.md file. + */ + addAgentInstruction(text: string): Promise + + /** + * Adds an agent skill and its supporting files to the sandbox. + */ + addAgentSkill(name: string, description: string, contents: string, options?: AgentSkillOptions): Promise + + /** + * Adds a custom agent and its supporting files to the sandbox. + */ + addCustomAgent(name: string, description: string, contents: string, options?: CustomAgentOptions): Promise + + /** + * Adds an MCP server to the sandbox's Copilot configuration. + */ + addMcpServer(name: string, config: McpServerConfig): Promise + + /** + * Installs a remote, local, or marketplace Copilot plugin in the sandbox. + */ + addCopilotPlugin(config: CopilotPluginConfig): Promise +} + +interface SandboxConstructor { + /** + * Creates a sandbox using the requested runtime options. + */ + create(options?: SandboxCreateOptions): Promise +} + +export type { + AgentSkillCopiedFile, + AgentSkillFile, + AgentSkillOptions, + AgentSkillWrittenFile, + CommandResult, + CopilotPluginConfig, + CopilotPluginSource, + CopyOptions, + CustomAgentCopiedFile, + CustomAgentFile, + CustomAgentOptions, + CustomAgentWrittenFile, + DownloadOptions, + LocalCopilotPluginSource, + McpServerConfig, + RemoteCopilotPluginSource, + RunOptions, + Sandbox, + SandboxConstructor, + SandboxCreateOptions, +} diff --git a/packages/agent-eval/src/sandbox/virtual.test.ts b/packages/agent-eval/src/sandbox/virtual.test.ts new file mode 100644 index 00000000..0fb8ea85 --- /dev/null +++ b/packages/agent-eval/src/sandbox/virtual.test.ts @@ -0,0 +1,85 @@ +import path from 'node:path' +import {describe, expect, test} from 'vitest' +import {VirtualHost} from '../host' +import {CONTAINER_WORKDIR} from './constants' +import {VirtualSandbox} from './virtual' + +describe('VirtualSandbox', () => { + test('reads, writes, and checks files in the sandbox workspace', async () => { + const host = VirtualHost.create() + const sandbox = await VirtualSandbox.create({host}) + + expect(await sandbox.exists('nested/example.txt')).toBe(false) + + await sandbox.writeFile('nested/example.txt', 'example') + + expect(await sandbox.exists('nested/example.txt')).toBe(true) + expect(await sandbox.readFile('nested/example.txt')).toBe('example') + expect(await host.fs.readFile(path.join(CONTAINER_WORKDIR, 'nested/example.txt'), 'utf8')).toBe('example') + }) + + test('resolves absolute and relative container paths', async () => { + const host = VirtualHost.create({ + '/absolute/example.txt': 'absolute', + [CONTAINER_WORKDIR]: { + 'relative.txt': 'relative', + }, + }) + const sandbox = await VirtualSandbox.create({host}) + + expect(await sandbox.readFile('/absolute/../absolute/example.txt')).toBe('absolute') + expect(await sandbox.readFile('./nested/../relative.txt')).toBe('relative') + expect(await sandbox.exists('/absolute/example.txt')).toBe(true) + }) + + test('copies host directories into the sandbox with exclusions', async () => { + const host = VirtualHost.create({ + '/fixture/included.txt': 'included', + '/fixture/nested/included.txt': 'nested', + '/fixture/nested/excluded.txt': 'excluded', + }) + const sandbox = await VirtualSandbox.create({host}) + + await sandbox.copy('/fixture', 'copied', { + exclude: ['nested/excluded.txt'], + }) + + expect(await sandbox.readFile('copied/included.txt')).toBe('included') + expect(await sandbox.readFile('copied/nested/included.txt')).toBe('nested') + expect(await sandbox.exists('copied/nested/excluded.txt')).toBe(false) + }) + + test('downloads sandbox directories to the host with ignored files', async () => { + const host = VirtualHost.create() + const sandbox = await VirtualSandbox.create({host}) + + await sandbox.writeFile('results/included.txt', 'included') + await sandbox.writeFile('results/nested/included.txt', 'nested') + await sandbox.writeFile('results/nested/ignored.txt', 'ignored') + + await sandbox.download('results', '/download', { + ignore(name) { + return name.endsWith('ignored.txt') + }, + }) + + expect(await host.fs.readFile('/download/included.txt', 'utf8')).toBe('included') + expect(await host.fs.readFile('/download/nested/included.txt', 'utf8')).toBe('nested') + await expect(host.fs.access('/download/nested/ignored.txt')).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + + test('downloads relative container files from the sandbox workspace', async () => { + const host = VirtualHost.create({ + [CONTAINER_WORKDIR]: { + 'result.txt': 'result', + }, + }) + const sandbox = await VirtualSandbox.create({host}) + + await sandbox.download('./nested/../result.txt', '/download') + + expect(await host.fs.readFile('/download/result.txt', 'utf8')).toBe('result') + }) +}) diff --git a/packages/agent-eval/src/sandbox/virtual.ts b/packages/agent-eval/src/sandbox/virtual.ts new file mode 100644 index 00000000..e84c1b40 --- /dev/null +++ b/packages/agent-eval/src/sandbox/virtual.ts @@ -0,0 +1,186 @@ +import path from 'node:path' +import {VirtualHost, type Host} from '../host' +import {resolveContainerPath} from './path' +import type {CommandResult, CopyOptions, DownloadOptions, Sandbox, SandboxCreateOptions} from './types' + +const defaultCreateOptions: SandboxCreateOptions = {} + +export class VirtualSandbox implements Sandbox { + static async create(options: SandboxCreateOptions = defaultCreateOptions) { + return new VirtualSandbox(options.host ?? VirtualHost.create()) + } + + [Symbol.asyncDispose](): Promise { + return Promise.resolve() + } + + #host: Host + + constructor(host: Host) { + this.#host = host + } + + async copy(sourcePath: string, destinationPath: string, options: CopyOptions = {}): Promise { + const source = path.resolve(sourcePath) + const sourceStats = await this.#host.fs.stat(source) + if (!sourceStats.isDirectory() && !sourceStats.isFile()) { + throw new Error(`Cannot copy "${sourcePath}" because it is not a file or directory`) + } + + const destination = resolveContainerPath(destinationPath) + if (!path.posix.basename(destination)) { + throw new Error(`Cannot copy "${sourcePath}" to "${destinationPath}" because the destination must include a name`) + } + + const excludedPaths = new Set(options.exclude?.map(filepath => normalizeExcludedPath(filepath, source))) + await copyPath(this.#host, source, destination, relativePath => { + return isExcluded(relativePath, excludedPaths) + }) + } + + async download(containerFilePath: string, hostDestinationPath: string, options: DownloadOptions = {}): Promise { + const source = resolveContainerPath(containerFilePath) + const destination = path.resolve(hostDestinationPath) + const sourceStats = await this.#host.fs.stat(source) + + await this.#host.fs.mkdir(destination, { + recursive: true, + }) + + if (sourceStats.isDirectory()) { + const entries = await this.#host.fs.readdir(source) + for (const entry of entries) { + const name = entry.toString() + if (options.ignore?.(name)) { + continue + } + + await copyPath(this.#host, path.posix.join(source, name), path.join(destination, name), relativePath => { + return options.ignore?.(path.posix.join(name, relativePath)) ?? false + }) + } + return + } + + const name = path.basename(source) + if (!options.ignore?.(name)) { + await copyPath(this.#host, source, path.join(destination, name)) + } + } + + readFile(filepath: string): Promise { + return this.#host.fs.readFile(resolveContainerPath(filepath), 'utf8') + } + + async writeFile(filepath: string, contents: string): Promise { + const destination = resolveContainerPath(filepath) + await this.#host.fs.mkdir(path.posix.dirname(destination), { + recursive: true, + }) + await this.#host.fs.writeFile(destination, contents, 'utf8') + } + + async exists(filepath: string): Promise { + try { + await this.#host.fs.access(resolveContainerPath(filepath)) + return true + } catch (error) { + if (isErrorWithCode(error, 'ENOENT')) { + return false + } + + throw error + } + } + + async runCommand(): Promise { + return { + stdout: '', + stderr: '', + exitCode: 0, + } + } + + async addAgentInstruction(): Promise {} + + async addAgentSkill(): Promise {} + + async addCustomAgent(): Promise {} + + async addMcpServer(): Promise {} + + async addCopilotPlugin(): Promise {} +} + +async function copyPath( + host: Host, + source: string, + destination: string, + ignore: (relativePath: string) => boolean = () => { + return false + }, + root: string = source, +): Promise { + const relativePath = normalizeCopyPath(path.relative(root, source)) + if (ignore(relativePath)) { + return + } + + const stats = await host.fs.stat(source) + if (stats.isDirectory()) { + await host.fs.mkdir(destination, { + recursive: true, + }) + + const entries = await host.fs.readdir(source) + for (const entry of entries) { + const name = entry.toString() + await copyPath(host, path.join(source, name), path.join(destination, name), ignore, root) + } + return + } + + if (!stats.isFile()) { + throw new Error(`Cannot copy "${source}" because it is not a file or directory`) + } + + await host.fs.mkdir(path.dirname(destination), { + recursive: true, + }) + await host.fs.copyFile(source, destination) +} + +function normalizeCopyPath(filepath: string): string { + const normalized = path.posix.normalize(filepath.split(path.sep).join(path.posix.sep)) + if (normalized === '.') { + return '' + } + + return normalized.replace(/\/$/, '') +} + +function normalizeExcludedPath(filepath: string, source: string): string { + if (path.isAbsolute(filepath)) { + return normalizeCopyPath(path.relative(source, filepath)) + } + + return normalizeCopyPath(filepath) +} + +function isExcluded(relativePath: string, excludedPaths: ReadonlySet): boolean { + if (!relativePath) { + return false + } + + for (const excludedPath of excludedPaths) { + if (relativePath === excludedPath || relativePath.startsWith(`${excludedPath}/`)) { + return true + } + } + + return false +} + +function isErrorWithCode(error: unknown, code: string): boolean { + return error instanceof Error && 'code' in error && error.code === code +} diff --git a/packages/agent-eval/src/scenario-config.ts b/packages/agent-eval/src/scenario-config.ts deleted file mode 100644 index 0a056418..00000000 --- a/packages/agent-eval/src/scenario-config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type {ScenarioConfig} from './experiment-config' - -function defineScenario(config: ScenarioConfig) { - return config -} - -export {defineScenario} -export type {ScenarioConfig} diff --git a/packages/agent-eval/src/scenario.test.ts b/packages/agent-eval/src/scenario.test.ts index 390a65a5..809d493a 100644 --- a/packages/agent-eval/src/scenario.test.ts +++ b/packages/agent-eval/src/scenario.test.ts @@ -1,120 +1,217 @@ -import fs from 'node:fs/promises' -import os from 'node:os' -import path from 'node:path' -import {afterEach, describe, expect, test} from 'vitest' -import {resolveExperimentScenario} from './resolve-experiment-scenario' - -const temporaryDirectories: Array = [] - -async function createTemporaryDirectory() { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-')) - temporaryDirectories.push(directory) - return directory -} - -afterEach(async () => { - await Promise.all( - temporaryDirectories.splice(0).map(directory => { - return fs.rm(directory, {recursive: true, force: true}) +import {test, expect} from 'vitest' +import {VirtualHost} from './host' +import {listScenarios, getScenario, defineConfig} from './scenario' + +test('listScenarios', async () => { + const config = JSON.stringify( + defineConfig({ + prompt: 'test', }), ) + const host = VirtualHost.create({ + '/scenarios': { + '.hidden-directory': {}, + '.hidden-file': '', + '001-scenario': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + '002-missing-package-json': { + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + '003-missing-config': { + 'package.json': '{}', + 'scenario.test.ts': '', + }, + '004-invalid-config': { + 'package.json': '{}', + 'scenario.config.ts': '', + }, + '005-missing-test': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + }, + }, + }) + const scenarios = await listScenarios(host, '/scenarios') + + expect(scenarios).toHaveLength(1) + expect(scenarios).toContainEqual({ + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'test', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }) + + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '.hidden-directory'})) + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '.hidden-file'})) + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '002-missing-package-json'})) + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '003-missing-config'})) + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '004-invalid-config'})) + expect(scenarios).not.toContainEqual(expect.objectContaining({id: '005-missing-test'})) }) -describe(resolveExperimentScenario, () => { - test('resolves named scenarios from the provided directory', async () => { - const scenariosDirectory = await createTemporaryDirectory() - const directory = path.join(scenariosDirectory, 'button-scenario') - await fs.mkdir(directory) - await fs.writeFile(path.join(directory, 'scenario.config.ts'), `export default {prompt: 'Use a button'}`) - await fs.writeFile(path.join(directory, 'scenario.test.ts'), '') - - await expect( - resolveExperimentScenario('button-scenario', { - directory: scenariosDirectory, - }), - ).resolves.toEqual({ - id: 'button-scenario', - directory, - config: { - prompt: 'Use a button', +test('listScenarios includes optional metadata and browser tests', async () => { + const config = JSON.stringify( + defineConfig({ + description: 'Test scenario', + prompt: 'Complete the task', + tags: ['test', 'browser'], + }), + ) + const host = VirtualHost.create({ + '/scenarios': { + '001-scenario': { + 'browser.test.ts': '', + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', }, - testPath: path.join(directory, 'scenario.test.ts'), - }) + }, }) - test('resolves inline scenario directories relative to the provided cwd', async () => { - const cwd = await createTemporaryDirectory() - const directory = path.join(cwd, 'scenarios', 'local-scenario') - await fs.mkdir(directory, {recursive: true}) - await fs.writeFile(path.join(directory, 'scenario.config.mjs'), `export default {prompt: 'Use ignored config'}`) - await fs.writeFile( - path.join(directory, 'scenario.config.ts'), - `export default {prompt: 'Update the local project'}`, - ) - await fs.writeFile(path.join(directory, 'scenario.test.ts'), '') - - await expect( - resolveExperimentScenario( - { - name: 'local-scenario', - path: 'scenarios/local-scenario', - }, - { - cwd, - }, - ), - ).resolves.toEqual({ - id: 'local-scenario', - directory, - config: { - prompt: 'Update the local project', + await expect(listScenarios(host, '/scenarios')).resolves.toEqual([ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + description: 'Test scenario', + tags: ['test', 'browser'], + testPath: '/scenarios/001-scenario/scenario.test.ts', + browserTestPath: '/scenarios/001-scenario/browser.test.ts', + }, + ]) +}) + +test('listScenarios sorts scenarios by directory name', async () => { + const config = JSON.stringify( + defineConfig({ + prompt: 'test', + }), + ) + const host = VirtualHost.create({ + '/scenarios': { + '002-last': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', }, - testPath: path.join(directory, 'scenario.test.ts'), - }) + '001-first': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + }, }) - test('defaults inline scenario names to the directory name', async () => { - const cwd = await createTemporaryDirectory() - const directory = path.join(cwd, 'scenarios', 'local-button-scenario') - await fs.mkdir(directory, {recursive: true}) - await fs.writeFile( - path.join(directory, 'scenario.config.ts'), - `export default {prompt: 'Update the local project'}`, - ) - await fs.writeFile(path.join(directory, 'scenario.test.ts'), '') - - await expect( - resolveExperimentScenario( - { - path: './scenarios/local-button-scenario', - }, - { - cwd, - }, - ), - ).resolves.toMatchObject({ - id: 'local-button-scenario', - directory, - }) + const scenarios = await listScenarios(host, '/scenarios') + + expect( + scenarios.map(scenario => { + return scenario.id + }), + ).toEqual(['001-first', '002-last']) +}) + +test('listScenarios ignores configs without a default export', async () => { + const config = JSON.stringify( + defineConfig({ + prompt: 'test', + }), + ) + const host = VirtualHost.create({ + '/scenarios': { + '001-scenario': { + 'package.json': '{}', + 'scenario.config.ts': `export const scenario = ${config}`, + 'scenario.test.ts': '', + }, + }, }) - test('requires inline scenarios to use the default scenario file structure', async () => { - const cwd = await createTemporaryDirectory() - const directory = path.join(cwd, 'fixtures', 'local-scenario') - await fs.mkdir(directory, {recursive: true}) - await fs.writeFile(path.join(directory, 'scenario.config.ts'), `export default {prompt: 'Use default config'}`) - await fs.writeFile(path.join(directory, 'custom.test.ts'), '') - - await expect( - resolveExperimentScenario( - { - name: 'local-scenario', - path: 'fixtures/local-scenario', - }, - { - cwd, - }, - ), - ).rejects.toThrow(`Scenario "local-scenario" test file was not found: ${path.join(directory, 'scenario.test.ts')}`) + await expect(listScenarios(host, '/scenarios')).resolves.toEqual([]) +}) + +test('listScenarios excludes template directories', async () => { + const config = JSON.stringify( + defineConfig({ + prompt: 'test', + }), + ) + const host = VirtualHost.create({ + '/scenarios': { + '000-template': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + }, + }) + + await expect(listScenarios(host, '/scenarios')).resolves.toEqual([]) +}) + +test('throws if input is not a directory', async () => { + const host = VirtualHost.create({ + '/test': '', + }) + + await expect(() => listScenarios(host, '/test')).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Expected scenarios path to be a directory]`, + ) +}) + +test('getScenario', async () => { + const config = JSON.stringify( + defineConfig({ + prompt: 'test', + }), + ) + const host = VirtualHost.create({ + '/scenarios': { + '001-scenario': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + '002-scenario': { + 'package.json': '{}', + 'scenario.config.ts': `export default ${config}`, + 'scenario.test.ts': '', + }, + }, + }) + + await expect(getScenario(host, '/scenarios', '001-scenario')).resolves.toEqual({ + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'test', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', }) + + await expect(getScenario(host, '/scenarios', '002-scenario')).resolves.toEqual({ + id: '002-scenario', + directory: '/scenarios/002-scenario', + prompt: 'test', + tags: [], + testPath: '/scenarios/002-scenario/scenario.test.ts', + }) + + await expect(getScenario(host, '/scenarios', '003-scenario')).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Scenario "003-scenario" was not found in: /scenarios]`, + ) +}) + +test('getScenario throws if input is not a directory', async () => { + const host = VirtualHost.create({ + '/test': '', + }) + + await expect(() => getScenario(host, '/test', '001-scenario')).rejects.toThrowErrorMatchingInlineSnapshot( + `[Error: Expected scenarios path to be a directory]`, + ) }) diff --git a/packages/agent-eval/src/scenario.ts b/packages/agent-eval/src/scenario.ts new file mode 100644 index 00000000..1b691ef6 --- /dev/null +++ b/packages/agent-eval/src/scenario.ts @@ -0,0 +1,185 @@ +import path from 'node:path' +import * as z from 'zod/mini' +import {DefaultHost, type Host} from './host' + +const ScenarioConfigSchema = z.object({ + description: z.optional(z.string()), + prompt: z.string(), + tags: z.optional(z.array(z.string())), +}) + +type ScenarioConfig = z.infer + +function defineConfig(config: ScenarioConfig): ScenarioConfig { + return config +} + +type ScenarioConfigModule = { + default?: unknown +} + +const ScenarioSchema = z.object({ + id: z.string(), + directory: z.string(), + prompt: z.string(), + description: z.optional(z.string()), + tags: z.array(z.string()), + testPath: z.string(), + browserTestPath: z.optional(z.string()), +}) + +type Scenario = z.infer + +type ScenarioSourceOptions = { + host?: Host + directory: string +} + +async function loadScenario(host: Host, directory: string, id = path.basename(directory)): Promise { + if (!host.existsSync(directory)) { + throw new Error(`Scenario "${id}" directory was not found: ${directory}`) + } + + const stats = await host.fs.stat(directory) + if (!stats.isDirectory()) { + throw new Error(`Scenario "${id}" directory was not found: ${directory}`) + } + + const configPath = path.join(directory, 'scenario.config.ts') + if (!host.existsSync(configPath)) { + throw new Error(`Scenario "${id}" config file was not found: ${configPath}`) + } + + const testPath = path.join(directory, 'scenario.test.ts') + if (!host.existsSync(testPath)) { + throw new Error(`Scenario "${id}" test file was not found: ${testPath}`) + } + + const data: ScenarioConfigModule = await host.loadModule(configPath) + const config = ScenarioConfigSchema.parse(data.default) + const scenario: Scenario = { + id, + directory, + prompt: config.prompt, + tags: config.tags ?? [], + testPath, + } + + if (config.description) { + scenario.description = config.description + } + + const browserTestPath = ['browser.test.ts', 'scenario.browser.test.ts'] + .map(filename => path.join(directory, filename)) + .find(filepath => host.existsSync(filepath)) + if (browserTestPath) { + scenario.browserTestPath = browserTestPath + } + + return scenario +} + +function getScenarioSource( + hostOrOptions: Host | ScenarioSourceOptions, + directory?: string, +): {host: Host; directory: string} { + if (directory !== undefined) { + return { + host: hostOrOptions as Host, + directory, + } + } + + const options = hostOrOptions as ScenarioSourceOptions + return { + host: options.host ?? DefaultHost, + directory: options.directory, + } +} + +async function listScenarios(options: ScenarioSourceOptions): Promise> +async function listScenarios(host: Host, directory: string): Promise> +async function listScenarios( + hostOrOptions: Host | ScenarioSourceOptions, + directory?: string, +): Promise> { + const source = getScenarioSource(hostOrOptions, directory) + const {host} = source + directory = source.directory + const stats = await host.fs.stat(directory) + if (!stats.isDirectory()) { + throw new Error('Expected scenarios path to be a directory') + } + + const entries = ( + await host.fs.readdir(directory, { + withFileTypes: true, + }) + ).sort((a, b) => { + return a.name.localeCompare(b.name) + }) + const candidates = entries.filter(entry => { + if (!entry.isDirectory()) { + return false + } + + const packageJsonPath = path.join(directory, entry.name, 'package.json') + if (!host.existsSync(packageJsonPath)) { + return false + } + + if (entry.name.startsWith('.')) { + return false + } + + if (entry.name.startsWith('000')) { + return false + } + + const scenarioConfigPath = path.join(directory, entry.name, 'scenario.config.ts') + if (!host.existsSync(scenarioConfigPath)) { + return false + } + + const testPath = path.join(directory, entry.name, 'scenario.test.ts') + if (!host.existsSync(testPath)) { + return false + } + + return true + }) + const scenarios: Array = [] + + for (const entry of candidates) { + const scenarioDirectory = path.join(directory, entry.name) + const data: ScenarioConfigModule = await host.loadModule(path.join(scenarioDirectory, 'scenario.config.ts')) + if (!ScenarioConfigSchema.safeParse(data.default).success) { + continue + } + + scenarios.push(await loadScenario(host, scenarioDirectory, entry.name)) + } + + return scenarios +} + +async function getScenario(options: ScenarioSourceOptions & {id: string}): Promise +async function getScenario(host: Host, directory: string, id: string): Promise +async function getScenario( + hostOrOptions: Host | (ScenarioSourceOptions & {id: string}), + directory?: string, + id?: string, +): Promise { + const source = getScenarioSource(hostOrOptions, directory) + id = id ?? (hostOrOptions as ScenarioSourceOptions & {id: string}).id + const scenarios = await listScenarios(source) + const scenario = scenarios.find(candidate => candidate.id === id) + if (scenario) { + return scenario + } + + throw new Error(`Scenario "${id}" was not found in: ${source.directory}`) +} + +export {defineConfig, listScenarios, getScenario, loadScenario, ScenarioSchema, ScenarioConfigSchema} +export type {ScenarioConfig, Scenario, ScenarioSourceOptions} diff --git a/packages/agent-eval/src/scenarios.test.ts b/packages/agent-eval/src/scenarios.test.ts deleted file mode 100644 index 0864c0c1..00000000 --- a/packages/agent-eval/src/scenarios.test.ts +++ /dev/null @@ -1,153 +0,0 @@ -import fs from 'node:fs/promises' -import os from 'node:os' -import path from 'node:path' -import {afterEach, describe, expect, test} from 'vitest' -import {findScenario, listScenarios} from './scenarios' - -const temporaryDirectories: Array = [] - -async function createScenariosDirectory() { - const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-scenarios-')) - temporaryDirectories.push(directory) - return directory -} - -async function createScenario( - scenariosDirectory: string, - id: string, - prompt: string, - tags?: Array, - description?: string, -) { - const directory = path.join(scenariosDirectory, id) - await fs.mkdir(directory) - await fs.writeFile( - path.join(directory, 'scenario.config.ts'), - `export default ${JSON.stringify({description, prompt, ...(tags ? {tags} : {})})}`, - ) - await fs.writeFile(path.join(directory, 'scenario.test.ts'), '') - return directory -} - -afterEach(async () => { - await Promise.all( - temporaryDirectories.splice(0).map(directory => { - return fs.rm(directory, {recursive: true, force: true}) - }), - ) -}) - -describe('scenario loading', () => { - test('lists scenarios from the provided directory', async () => { - const scenariosDirectory = await createScenariosDirectory() - await createScenario(scenariosDirectory, 'second', 'Second prompt') - await createScenario(scenariosDirectory, 'first', 'First prompt') - await fs.writeFile(path.join(scenariosDirectory, 'README.md'), '') - - await expect(listScenarios({directory: scenariosDirectory})).resolves.toEqual([ - expect.objectContaining({id: 'first', config: {prompt: 'First prompt'}}), - expect.objectContaining({id: 'second', config: {prompt: 'Second prompt'}}), - ]) - }) - - test('loads scenario descriptions', async () => { - const scenariosDirectory = await createScenariosDirectory() - await createScenario(scenariosDirectory, 'example', 'Example prompt', undefined, 'Example description') - - await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toMatchObject({ - config: { - description: 'Example description', - prompt: 'Example prompt', - }, - }) - }) - - test('rejects non-string scenario descriptions', async () => { - const scenariosDirectory = await createScenariosDirectory() - const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt') - await fs.writeFile( - path.join(directory, 'scenario.config.ts'), - `export default {description: 42, prompt: 'Example prompt'}`, - ) - - await expect(findScenario('example', {directory: scenariosDirectory})).rejects.toThrow( - 'Scenario "example" config must export a default config with a string prompt, optional string description, and optional string[] tags', - ) - }) - - test('lists scenarios that match all provided tags', async () => { - const scenariosDirectory = await createScenariosDirectory() - await createScenario(scenariosDirectory, 'both', 'Both tags', ['baseline', 'primer']) - await createScenario(scenariosDirectory, 'baseline', 'Baseline only', ['baseline']) - await createScenario(scenariosDirectory, 'untagged', 'No tags') - - await expect(listScenarios({directory: scenariosDirectory, tags: ['baseline', 'primer']})).resolves.toEqual([ - expect.objectContaining({id: 'both'}), - ]) - }) - - test('lists all scenarios when no tags are provided', async () => { - const scenariosDirectory = await createScenariosDirectory() - await createScenario(scenariosDirectory, 'tagged', 'Tagged', ['baseline']) - await createScenario(scenariosDirectory, 'untagged', 'Untagged') - - await expect(listScenarios({directory: scenariosDirectory, tags: []})).resolves.toEqual([ - expect.objectContaining({id: 'tagged'}), - expect.objectContaining({id: 'untagged'}), - ]) - }) - - test('finds a scenario by id in the provided directory', async () => { - const scenariosDirectory = await createScenariosDirectory() - const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt') - - await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({ - id: 'example', - directory, - config: {prompt: 'Example prompt'}, - testPath: path.join(directory, 'scenario.test.ts'), - }) - }) - - test('includes an optional browser test', async () => { - const scenariosDirectory = await createScenariosDirectory() - const directory = await createScenario(scenariosDirectory, 'example', 'Example prompt') - const browserTestPath = path.join(directory, 'scenario.browser.test.ts') - await fs.writeFile(browserTestPath, '') - - await expect(findScenario('example', {directory: scenariosDirectory})).resolves.toEqual({ - id: 'example', - directory, - config: {prompt: 'Example prompt'}, - testPath: path.join(directory, 'scenario.test.ts'), - browserTestPath, - }) - }) - - test('returns undefined when a scenario is not found', async () => { - const scenariosDirectory = await createScenariosDirectory() - - await expect(findScenario('missing-scenario', {directory: scenariosDirectory})).resolves.toBeUndefined() - }) - - test('returns undefined when a scenario id is not a direct child directory', async () => { - const scenariosDirectory = await createScenariosDirectory() - const siblingDirectory = await createScenariosDirectory() - await createScenario(siblingDirectory, 'example', 'Example prompt') - - await expect( - findScenario(path.relative(scenariosDirectory, path.join(siblingDirectory, 'example')), { - directory: scenariosDirectory, - }), - ).resolves.toBeUndefined() - }) - - test('throws when the scenarios directory does not exist', async () => { - const scenariosDirectory = await createScenariosDirectory() - const missingDirectory = path.join(scenariosDirectory, 'missing') - - await expect(findScenario('example', {directory: missingDirectory})).rejects.toThrow( - `Scenarios directory does not exist: ${missingDirectory}`, - ) - }) -}) diff --git a/packages/agent-eval/src/scenarios.ts b/packages/agent-eval/src/scenarios.ts deleted file mode 100644 index 18a56865..00000000 --- a/packages/agent-eval/src/scenarios.ts +++ /dev/null @@ -1,128 +0,0 @@ -import fs from 'node:fs/promises' -import path from 'node:path' -import {pathToFileURL} from 'node:url' -import type {ScenarioConfig} from './experiment-config' - -type ResolvedScenario = { - readonly id: string - readonly directory: string - readonly config: ScenarioConfig - readonly testPath: string - readonly browserTestPath?: string -} - -type ScenarioSourceOptions = { - directory?: string - tags?: ReadonlyArray -} - -function resolveScenariosDirectory(options: ScenarioSourceOptions): string { - return path.resolve(options.directory ?? 'scenarios') -} - -async function assertScenariosDirectory(directory: string) { - const stats = await fs.stat(directory).catch(() => undefined) - if (!stats) { - throw new Error(`Scenarios directory does not exist: ${directory}`) - } - if (!stats.isDirectory()) { - throw new Error(`Scenarios path is not a directory: ${directory}`) - } -} - -async function assertScenarioDirectory(directory: string, name: string) { - const stats = await fs.stat(directory).catch(() => undefined) - if (!stats?.isDirectory()) { - throw new Error(`Scenario "${name}" directory was not found: ${directory}`) - } -} - -async function assertScenarioFile(filepath: string, name: string, kind: 'config' | 'test') { - const stats = await fs.stat(filepath).catch(() => undefined) - if (!stats?.isFile()) { - throw new Error(`Scenario "${name}" ${kind} file was not found: ${filepath}`) - } -} - -function isScenarioConfig(value: unknown): value is ScenarioConfig { - if (value === null || typeof value !== 'object') { - return false - } - - const config = value as Record - return ( - typeof config.prompt === 'string' && - (config.description === undefined || typeof config.description === 'string') && - (config.tags === undefined || - (Array.isArray(config.tags) && config.tags.every((tag: unknown) => typeof tag === 'string'))) - ) -} - -async function loadScenarioConfig(configPath: string, name: string): Promise { - const configModule = (await import(pathToFileURL(configPath).href)) as {default?: unknown} - if (!isScenarioConfig(configModule.default)) { - throw new Error( - `Scenario "${name}" config must export a default config with a string prompt, optional string description, and optional string[] tags`, - ) - } - return configModule.default -} - -async function loadScenarioDirectory(directory: string, name = path.basename(directory)): Promise { - await assertScenarioDirectory(directory, name) - - const configPath = path.join(directory, 'scenario.config.ts') - const testPath = path.join(directory, 'scenario.test.ts') - const browserTestPath = path.join(directory, 'scenario.browser.test.ts') - await assertScenarioFile(configPath, name, 'config') - await assertScenarioFile(testPath, name, 'test') - const browserTestStats = await fs.stat(browserTestPath).catch(() => undefined) - - return { - id: name, - directory, - config: await loadScenarioConfig(configPath, name), - testPath, - ...(browserTestStats?.isFile() ? {browserTestPath} : {}), - } -} - -async function getScenarioDirectoryNames(options: ScenarioSourceOptions): Promise> { - const scenariosDirectory = resolveScenariosDirectory(options) - await assertScenariosDirectory(scenariosDirectory) - - const entries = await fs.readdir(scenariosDirectory, {withFileTypes: true}) - return entries - .filter(entry => entry.isDirectory()) - .map(entry => entry.name) - .toSorted() -} - -async function listScenarios(options: ScenarioSourceOptions = {}): Promise> { - const scenariosDirectory = resolveScenariosDirectory(options) - const names = await getScenarioDirectoryNames(options) - const scenarios = await Promise.all( - names.map(name => loadScenarioDirectory(path.join(scenariosDirectory, name), name)), - ) - return scenarios.filter(scenario => options.tags?.every(tag => scenario.config.tags?.includes(tag)) ?? true) -} - -async function findScenario(id: string, options: ScenarioSourceOptions = {}): Promise { - const scenariosDirectory = resolveScenariosDirectory(options) - await assertScenariosDirectory(scenariosDirectory) - - const directory = path.resolve(scenariosDirectory, id) - if (path.dirname(directory) !== scenariosDirectory) { - return undefined - } - - const stats = await fs.stat(directory).catch(() => undefined) - if (!stats?.isDirectory()) { - return undefined - } - - return loadScenarioDirectory(directory, id) -} - -export {findScenario, listScenarios, loadScenarioDirectory} -export type {ResolvedScenario, ScenarioSourceOptions} diff --git a/packages/agent-eval/src/treatment.test.ts b/packages/agent-eval/src/treatment.test.ts new file mode 100644 index 00000000..f57c69fb --- /dev/null +++ b/packages/agent-eval/src/treatment.test.ts @@ -0,0 +1,33 @@ +import {describe, expect, test, vi} from 'vitest' +import {VirtualSandbox} from './sandbox' +import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema} from './treatment' + +describe('TreatmentSchema', () => { + test('parses the control treatment', () => { + expect(TreatmentSchema.parse(ControlTreatment)).toEqual({ + name: 'Control', + }) + }) + + test('parses and runs a setup function', async () => { + const setup = vi.fn(async () => {}) + const treatment = TreatmentSchema.parse({ + name: 'Example', + setup, + }) + const sandbox = await VirtualSandbox.create() + + await treatment.setup?.({sandbox}) + + expect(setup).toHaveBeenCalledWith({sandbox}) + }) + + test('rejects invalid setup return values', async () => { + const setup = TreatmentSetupSchema.parse(() => { + return 'invalid' + }) + const sandbox = await VirtualSandbox.create() + + await expect(setup({sandbox})).rejects.toThrow() + }) +}) diff --git a/packages/agent-eval/src/treatment.ts b/packages/agent-eval/src/treatment.ts index 40dfb674..e453eb3b 100644 --- a/packages/agent-eval/src/treatment.ts +++ b/packages/agent-eval/src/treatment.ts @@ -1,56 +1,27 @@ -import type {ExperimentConfig, TreatmentConfig} from './experiment-config' -import type {Model, ReasoningEffort} from './model' -import type {Message} from './copilot-cli' -import type {ResolvedScenario} from './resolve-experiment-scenario' +import * as z from 'zod/mini' +import {SandboxSchema} from './sandbox' -type Treatment = { - config: TreatmentConfig - scenario: ResolvedScenario - experiment: ExperimentConfig - id: string - model: Model - reasoningEffort?: ReasoningEffort -} +const TreatmentSetupSchema = z.function({ + input: [ + z.object({ + sandbox: SandboxSchema, + }), + ], + output: z.promise(z.void()), +}) -type TreatmentResult = { - id: string - treatment: Treatment - artifacts: { - copilotConfigPath: string - directory: string - skillsConfigPath: string - testResultsPath: string - workspacePath: string - } - assistant: { - logs: Array - turns: number - outputTokens: number - premiumRequests: number - totalApiDurationMs: number - sessionDurationMs: number - tools: Record - } - testResults: { - numTotalTests: number - numPassedTests: number - numFailedTests: number - numPendingTests: number - numTodoTests: number - tests: Array<{ - title: string - fullName: string - status: 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' - description?: string - }> - } - walkthrough: Walkthrough -} +type TreatmentSetup = z.infer + +const TreatmentSchema = z.object({ + name: z.string(), + setup: z.optional(TreatmentSetupSchema), +}) -type Walkthrough = - | {type: 'Unavailable'} - | {type: 'Screenshot'; filepath: string} - | {type: 'Screenshots'; screenshots: Array} - | {type: 'Video'; filepath: string} +type Treatment = z.infer + +const ControlTreatment: Treatment = { + name: 'Control', +} -export type {Treatment, TreatmentResult, Walkthrough} +export {ControlTreatment, TreatmentSchema, TreatmentSetupSchema} +export type {Treatment, TreatmentSetup} diff --git a/packages/agent-eval/src/trial.test.ts b/packages/agent-eval/src/trial.test.ts new file mode 100644 index 00000000..6d6e6552 --- /dev/null +++ b/packages/agent-eval/src/trial.test.ts @@ -0,0 +1,1026 @@ +import path from 'node:path' +import {describe, expect, test, vi} from 'vitest' +import {VirtualHost, type Host} from './host' +import { + AGENTS_DIR, + CONTAINER_WORKDIR, + COPILOT_DIR, + NODE_USER, + SKILLS_DIR, + type CommandResult, + type Sandbox, +} from './sandbox' +import {run} from './trial' +import type {Trial} from './trial' +import type {ResultMessage} from './copilot-cli' + +async function setup(trial: Trial) { + const artifactsDirectory = '/artifacts' + const host = VirtualHost.create({ + [AGENTS_DIR]: {}, + [COPILOT_DIR]: {}, + [artifactsDirectory]: {}, + [trial.scenario.directory]: { + '.next': { + 'build.txt': '', + }, + node_modules: { + 'dependency.txt': '', + }, + 'scenario.config.ts': '', + 'scenario.test.ts': '', + ...(trial.scenario.browserTestPath ? {[path.basename(trial.scenario.browserTestPath)]: ''} : {}), + }, + }) + const sandbox = await host.createSandbox() + + vi.spyOn(sandbox, 'copy') + + return { + artifactsDirectory, + copilotToken: '', + host, + sandbox, + } +} + +type RunCommandMockOptions = { + params: Parameters + sandbox: Sandbox +} +type RunCommandMock = (options: RunCommandMockOptions) => Promise + +async function applyCommandMocks( + mocks: Array, + options: RunCommandMockOptions, +): Promise { + for (const mock of mocks) { + const result = await mock(options) + if (result) { + return result + } + } +} + +const writeCopilotResult: RunCommandMock = async ({params}) => { + const [command, args] = params + if (command === 'copilot' && Array.isArray(args) && args[0] === '--prompt') { + const result: ResultMessage = { + type: 'result', + timestamp: '', + sessionId: '', + exitCode: 0, + usage: { + premiumRequests: 0, + totalApiDurationMs: 0, + sessionDurationMs: 0, + codeChanges: { + linesAdded: 0, + linesRemoved: 0, + filesModified: [], + }, + }, + } + return { + stdout: [JSON.stringify(result)].join('\n'), + stderr: '', + exitCode: 0, + } + } +} + +const writeTestFile: RunCommandMock = async ({params, sandbox}) => { + const [command, args] = params + if (command === 'sh' && Array.isArray(args) && args[0] === '-c' && args[1].startsWith('npx vitest run')) { + const config = await sandbox.readFile('vitest.agent-eval.config.ts') + const outputFile = config.match(/outputFile: "([^"]+)"/)?.[1] + if (!outputFile) { + throw new Error('Vitest output file was not configured') + } + + await sandbox.writeFile( + outputFile, + JSON.stringify({ + numTotalTests: 0, + numPassedTests: 0, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }), + ) + } +} + +function createTrial(): Trial { + return { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } +} + +function mockRunCommand(sandbox: Sandbox, mocks: Array = []) { + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([...mocks, writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) +} + +function writeWalkthroughArtifact(filepath: string, contents = ''): RunCommandMock { + return async ({params, sandbox}) => { + const [command, args] = params + if ( + command === 'copilot' && + Array.isArray(args) && + args[0] === '--prompt' && + args[1].startsWith('Record a visual walkthrough') + ) { + await sandbox.writeFile(filepath, contents) + } + } +} + +function manageAgentBrowserSkill(host: Host): RunCommandMock { + return async ({params, sandbox}) => { + const [command, args] = params + const skillDirectory = path.posix.join(SKILLS_DIR, 'agent-browser') + + if (command === 'npx' && Array.isArray(args) && args[0] === 'skills' && args[1] === 'add') { + await sandbox.writeFile(path.posix.join(skillDirectory, 'SKILL.md'), 'agent browser skill') + } + + if (command === 'rm' && Array.isArray(args) && args[0] === '-rf' && args[1] === skillDirectory) { + await host.fs.rm(skillDirectory, {recursive: true, force: true}) + } + } +} + +describe('run', () => { + test('collects output tokens from model messages without double counting assistant messages', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + const writeTokenOutput: RunCommandMock = async ({params}) => { + const [command, args] = params + if (command !== 'copilot' || !Array.isArray(args) || args[0] !== '--prompt') { + return + } + + return { + stdout: [ + JSON.stringify({ + type: 'model.message', + data: { + message: { + role: 'assistant', + outputTokens: 42, + }, + }, + ephemeral: true, + id: 'model-message', + timestamp: '', + parentId: '', + }), + JSON.stringify({ + type: 'assistant.message', + data: { + messageId: 'assistant-message', + content: 'Done.', + toolRequests: [], + interactionId: 'interaction', + turnId: 'turn', + outputTokens: 42, + }, + id: 'assistant-message', + timestamp: '', + parentId: '', + }), + JSON.stringify({ + type: 'result', + timestamp: '', + sessionId: '', + exitCode: 0, + usage: { + premiumRequests: 0, + totalApiDurationMs: 0, + sessionDurationMs: 0, + codeChanges: { + linesAdded: 0, + linesRemoved: 0, + filesModified: [], + }, + }, + }), + ].join('\n'), + stderr: '', + exitCode: 0, + } + } + mockRunCommand(sandbox, [writeTokenOutput]) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(result.agent.sessions[0].outputTokens).toBe(42) + }) + + test('copies scenario files into the container workdir', async () => { + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.directory, CONTAINER_WORKDIR, { + exclude: [ + 'scenario.config.ts', + 'scenario.test.ts', + 'browser.test.ts', + 'scenario.browser.test.ts', + 'node_modules', + '.next', + 'dist', + ], + }) + }) + + test('sets permissions for the scenario files', async () => { + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('chown', ['-R', NODE_USER, '.'], { + user: 'root', + }) + }) + + test('obfuscates the package name', async () => { + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['pkg', 'set', `name=${trial.id}`], { + user: NODE_USER, + }) + }) + + test('removes @primer/agent-eval from the workspace dependencies', async () => { + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], { + user: NODE_USER, + }) + }) + + test('installs workspace dependencies', async () => { + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['install'], { + user: NODE_USER, + }) + }) + + test('runs the generic setup', async () => { + const genericSetup = vi.fn(async () => { + // + }) + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + setup: genericSetup, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(genericSetup).toHaveBeenCalledWith({ + sandbox, + }) + }) + + test('runs the treatment setup', async () => { + const treatmentSetup = vi.fn(async () => { + // + }) + const trial: Trial = { + id: 'test-id', + scenario: { + id: 'test-id', + directory: '/scenarios/test', + prompt: 'test-prompt', + tags: [], + testPath: '/scenarios/test/scenario.test.ts', + }, + treatment: { + name: 'test-treatment-name', + setup: treatmentSetup, + }, + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + } + const {sandbox, ...runOptions} = await setup(trial) + const runCommand = sandbox.runCommand + + vi.spyOn(sandbox, 'runCommand').mockImplementation(async (command, args, commandOptions) => { + const result = await applyCommandMocks([writeTestFile, writeCopilotResult], { + params: [command, args, commandOptions], + sandbox, + }) + if (result) { + return result + } + return runCommand(command, args, commandOptions) + }) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(treatmentSetup).toHaveBeenCalledWith({ + sandbox, + }) + }) + + test('runs the build script when one exists', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + await host.fs.writeFile( + path.join(trial.scenario.directory, 'package.json'), + JSON.stringify({ + scripts: { + build: 'build', + }, + }), + ) + mockRunCommand(sandbox) + + await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith('npm', ['run', 'build', '--if-present'], { + user: NODE_USER, + }) + }) + + test('continues when no build script exists', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + mockRunCommand(sandbox) + + await expect( + run({ + ...runOptions, + sandbox, + trial, + }), + ).resolves.toMatchObject({ + trial, + }) + }) + + test('runs Copilot with the trial arguments', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + const copilotToken = 'test-token' + mockRunCommand(sandbox) + + await run({ + ...runOptions, + copilotToken, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith( + 'copilot', + [ + '--prompt', + trial.scenario.prompt, + '--model', + trial.model.name, + '--reasoning-effort', + trial.model.reasoningEffort, + '--mode', + 'autopilot', + '--allow-all', + '--output-format', + 'json', + ], + { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + }, + ) + }) + + test('runs the scenario tests', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + vi.spyOn(sandbox, 'writeFile') + mockRunCommand(sandbox) + + await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.testPath, 'scenario.test.ts') + expect(sandbox.writeFile).toHaveBeenCalledWith( + 'vitest.agent-eval.config.ts', + expect.stringContaining('outputFile: "test-results.json"'), + ) + expect(sandbox.runCommand).toHaveBeenCalledWith( + 'sh', + [ + '-c', + 'npx vitest run --config "$1" "$2" || true', + 'vitest-run', + 'vitest.agent-eval.config.ts', + 'scenario.test.ts', + ], + { + user: NODE_USER, + env: {}, + }, + ) + }) + + test('runs and combines browser tests with scenario tests', async () => { + const trial = createTrial() + trial.scenario.browserTestPath = '/scenarios/test/scenario.browser.test.ts' + const {sandbox, ...runOptions} = await setup(trial) + const writeTestResults: RunCommandMock = async ({params, sandbox: testSandbox}) => { + const [command, args] = params + if (command !== 'sh' || !Array.isArray(args) || args[0] !== '-c') { + return + } + + const browser = args.at(-1) === 'scenario.browser.test.ts' + const outputFile = browser ? 'browser-test-results.json' : 'test-results.json' + await testSandbox.writeFile( + outputFile, + JSON.stringify({ + numTotalTests: 1, + numPassedTests: browser ? 0 : 1, + numFailedTests: browser ? 1 : 0, + numPendingTests: 0, + numTodoTests: 0, + success: !browser, + testResults: [ + { + assertionResults: [], + }, + ], + }), + ) + + return { + stdout: '', + stderr: '', + exitCode: 0, + } + } + mockRunCommand(sandbox, [writeTestResults]) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith( + 'npm', + ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'], + { + user: NODE_USER, + }, + ) + expect(sandbox.runCommand).toHaveBeenCalledWith( + './node_modules/.bin/playwright', + ['install', '--with-deps', 'chromium'], + { + user: 'root', + env: { + PLAYWRIGHT_BROWSERS_PATH: '/ms-playwright', + }, + }, + ) + expect(sandbox.copy).toHaveBeenCalledWith(trial.scenario.browserTestPath, 'scenario.browser.test.ts') + expect(sandbox.runCommand).toHaveBeenCalledWith( + 'sh', + [ + '-c', + 'npx vitest run --config "$1" "$2" || true', + 'vitest-run', + 'vitest.agent-eval.config.ts', + 'scenario.browser.test.ts', + ], + { + user: NODE_USER, + env: { + PLAYWRIGHT_BROWSERS_PATH: '/ms-playwright', + }, + }, + ) + expect(result.testResults).toMatchObject({ + numTotalTests: 2, + numPassedTests: 1, + numFailedTests: 1, + success: false, + }) + }) + + test('runs Copilot with the walkthrough arguments', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + const copilotToken = 'test-token' + mockRunCommand(sandbox) + + await run({ + ...runOptions, + copilotToken, + sandbox, + trial, + }) + + expect(sandbox.runCommand).toHaveBeenCalledWith( + 'copilot', + [ + '--prompt', + expect.stringContaining('Record a visual walkthrough'), + '--model', + 'gpt-5.6-terra', + '--reasoning-effort', + 'medium', + '--mode', + 'autopilot', + '--allow-all', + '--output-format', + 'json', + ], + { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + allowNonZeroExitCode: true, + }, + ) + }) + + describe('walkthrough artifacts', () => { + test('returns unavailable when no walkthrough artifact exists', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + mockRunCommand(sandbox) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(result.walkthrough).toEqual({ + type: 'Unavailable', + }) + }) + + test('returns a video walkthrough', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/walkthrough.webm')]) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(result.walkthrough).toEqual({ + type: 'Video', + filepath: '/artifacts/test-id/walkthrough/walkthrough.webm', + }) + }) + + test('returns multiple walkthrough screenshots', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + const writeScreenshots: RunCommandMock = async options => { + await writeWalkthroughArtifact('walkthrough/screenshots/10.png')(options) + await writeWalkthroughArtifact('walkthrough/screenshots/2.jpg')(options) + await writeWalkthroughArtifact('walkthrough/screenshots/01.jpeg')(options) + await writeWalkthroughArtifact('walkthrough/screenshots/notes.txt')(options) + } + mockRunCommand(sandbox, [writeScreenshots]) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(result.walkthrough).toEqual({ + type: 'Screenshots', + screenshots: [ + '/artifacts/test-id/walkthrough/screenshots/01.jpeg', + '/artifacts/test-id/walkthrough/screenshots/2.jpg', + '/artifacts/test-id/walkthrough/screenshots/10.png', + ], + }) + }) + + test('returns a single walkthrough screenshot', async () => { + const trial = createTrial() + const {sandbox, ...runOptions} = await setup(trial) + mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/screenshot.png')]) + + const result = await run({ + ...runOptions, + sandbox, + trial, + }) + + expect(result.walkthrough).toEqual({ + type: 'Screenshot', + filepath: '/artifacts/test-id/walkthrough/screenshot.png', + }) + }) + }) + + describe('artifacts', () => { + test('creates the artifact directory when it does not exist', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + mockRunCommand(sandbox) + + expect(host.existsSync('/artifacts/test-id')).toBe(false) + + await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + expect(host.existsSync('/artifacts/test-id/workspace')).toBe(true) + }) + + test('empties the artifact directory when it already exists', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + await host.fs.mkdir('/artifacts/test-id', { + recursive: true, + }) + await host.fs.writeFile('/artifacts/test-id/stale.txt', 'stale') + mockRunCommand(sandbox) + + await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + expect(host.existsSync('/artifacts/test-id/stale.txt')).toBe(false) + expect(host.existsSync('/artifacts/test-id/workspace')).toBe(true) + }) + + test('downloads the workspace results', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + mockRunCommand(sandbox) + + const result = await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + await expect(host.fs.readFile(result.artifacts.testResultsPath, 'utf8')).resolves.toBe( + JSON.stringify({ + numTotalTests: 0, + numPassedTests: 0, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + success: true, + testResults: [], + }), + ) + }) + + test('downloads the Copilot configuration', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + await host.fs.writeFile(path.join(COPILOT_DIR, 'config.json'), 'copilot config') + mockRunCommand(sandbox) + + const result = await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + await expect( + host.fs.readFile(path.join(result.artifacts.copilotConfigDirectory, 'config.json'), 'utf8'), + ).resolves.toBe('copilot config') + }) + + test('downloads the agent configuration', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + await host.fs.writeFile(path.join(AGENTS_DIR, 'AGENTS.md'), 'agent config') + mockRunCommand(sandbox) + + const result = await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + await expect( + host.fs.readFile(path.join(result.artifacts.skillsConfigDirectory, 'AGENTS.md'), 'utf8'), + ).resolves.toBe('agent config') + }) + + test('excludes the walkthrough skill from the agent configuration', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + const treatmentSkillPath = path.posix.join(SKILLS_DIR, 'treatment-skill', 'SKILL.md') + await host.fs.mkdir(path.posix.dirname(treatmentSkillPath), {recursive: true}) + await host.fs.writeFile(treatmentSkillPath, 'treatment skill') + mockRunCommand(sandbox, [manageAgentBrowserSkill(host)]) + + const result = await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + expect(host.existsSync(path.join(result.artifacts.skillsConfigDirectory, 'skills', 'agent-browser'))).toBe(false) + await expect( + host.fs.readFile( + path.join(result.artifacts.skillsConfigDirectory, 'skills', 'treatment-skill', 'SKILL.md'), + 'utf8', + ), + ).resolves.toBe('treatment skill') + }) + + test('downloads the walkthrough', async () => { + const trial = createTrial() + const {sandbox, host, ...runOptions} = await setup(trial) + mockRunCommand(sandbox, [writeWalkthroughArtifact('walkthrough/screenshot.png', 'screenshot')]) + + const result = await run({ + ...runOptions, + host, + sandbox, + trial, + }) + + expect(result.walkthrough).toEqual({ + type: 'Screenshot', + filepath: '/artifacts/test-id/walkthrough/screenshot.png', + }) + await expect(host.fs.readFile('/artifacts/test-id/walkthrough/screenshot.png', 'utf8')).resolves.toBe( + 'screenshot', + ) + expect(host.existsSync('/artifacts/test-id/workspace/walkthrough')).toBe(false) + }) + }) +}) diff --git a/packages/agent-eval/src/trial.ts b/packages/agent-eval/src/trial.ts new file mode 100644 index 00000000..90b04335 --- /dev/null +++ b/packages/agent-eval/src/trial.ts @@ -0,0 +1,571 @@ +import path from 'node:path' +import {isMessageType, MessageSchema, parseMessage, type Message} from './copilot-cli' +import {DefaultHost, type Host} from './host' +import {AGENTS_DIR, CONTAINER_WORKDIR, COPILOT_DIR, NODE_USER, SKILLS_DIR, type Sandbox} from './sandbox' +import {parseTestResults, TestResultsSchema} from './vitest' +import {logger} from './logger' +import * as z from 'zod/mini' +import {ModelVariantSchema} from './model' +import {ScenarioSchema} from './scenario' +import {TreatmentSchema, TreatmentSetupSchema} from './treatment' + +const TrialSchema = z.object({ + id: z.string(), + scenario: ScenarioSchema, + treatment: TreatmentSchema, + model: ModelVariantSchema, + setup: z.optional(TreatmentSetupSchema), +}) + +type Trial = z.infer + +const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg']) +const AGENT_BROWSER_SKILL_DIRECTORY = path.posix.join(SKILLS_DIR, 'agent-browser') +const PLAYWRIGHT_BROWSERS_PATH = '/ms-playwright' + +const WalkthroughSchema = z.discriminatedUnion('type', [ + z.object({type: z.literal('Unavailable')}), + z.object({type: z.literal('Screenshot'), filepath: z.string()}), + z.object({type: z.literal('Screenshots'), screenshots: z.array(z.string())}), + z.object({type: z.literal('Video'), filepath: z.string()}), +]) + +type Walkthrough = z.infer + +const AgentSessionSchema = z.object({ + turns: z.number(), + outputTokens: z.number(), + premiumRequests: z.number(), + totalApiDurationMs: z.number(), + sessionDurationMs: z.number(), + tools: z.record(z.string(), z.number()), + messages: z.array(MessageSchema), +}) + +type AgentSession = z.infer + +const TrialArtifactsSchema = z.object({ + directory: z.string(), + copilotConfigDirectory: z.string(), + skillsConfigDirectory: z.string(), + testResultsPath: z.string(), + workspaceDirectory: z.string(), +}) + +const TrialAgentSchema = z.object({ + sessions: z.array(AgentSessionSchema), +}) + +const TrialResultSchema = z.object({ + artifacts: TrialArtifactsSchema, + trial: TrialSchema, + agent: TrialAgentSchema, + testResults: TestResultsSchema, + walkthrough: WalkthroughSchema, +}) + +type TrialResult = z.infer + +type PortableTrialPaths = Pick + +function getPortableTrialPaths(result: TrialResult, baseDirectory: string): PortableTrialPaths { + const toPortablePath = (filepath: string): string => { + if (!path.isAbsolute(filepath)) { + return filepath.split(path.sep).join(path.posix.sep) + } + + return path.relative(baseDirectory, filepath).split(path.sep).join(path.posix.sep) + } + + let walkthrough: Walkthrough + if (result.walkthrough.type === 'Screenshots') { + walkthrough = { + type: 'Screenshots', + screenshots: result.walkthrough.screenshots.map(toPortablePath), + } + } else if (result.walkthrough.type === 'Screenshot' || result.walkthrough.type === 'Video') { + walkthrough = { + ...result.walkthrough, + filepath: toPortablePath(result.walkthrough.filepath), + } + } else { + walkthrough = result.walkthrough + } + + return { + artifacts: { + directory: toPortablePath(result.artifacts.directory), + copilotConfigDirectory: toPortablePath(result.artifacts.copilotConfigDirectory), + skillsConfigDirectory: toPortablePath(result.artifacts.skillsConfigDirectory), + testResultsPath: toPortablePath(result.artifacts.testResultsPath), + workspaceDirectory: toPortablePath(result.artifacts.workspaceDirectory), + }, + walkthrough, + } +} + +async function run({ + artifactsDirectory, + copilotToken, + host = DefaultHost, + sandbox, + trial, +}: { + artifactsDirectory: string + copilotToken: string + host?: Host + sandbox: Sandbox + trial: Trial +}): Promise { + const logPrefix = `[${trial.scenario.id}] [${trial.treatment.name}] [${trial.model.name} (${trial.model.reasoningEffort})]` + + logger.info('%s Running trial: %s', logPrefix, trial.id) + + logger.info('%s Copying files from: %s...', logPrefix, trial.scenario.directory) + + await sandbox.copy(trial.scenario.directory, CONTAINER_WORKDIR, { + exclude: [ + 'scenario.config.ts', + 'scenario.test.ts', + 'browser.test.ts', + 'scenario.browser.test.ts', + 'node_modules', + '.next', + 'dist', + ], + }) + await sandbox.runCommand('chown', ['-R', NODE_USER, '.'], { + user: 'root', + }) + + logger.info('%s Obfuscating package name...', logPrefix) + await sandbox.runCommand('npm', ['pkg', 'set', `name=${trial.id}`], { + user: NODE_USER, + }) + + logger.info('%s Removing workspace dependency...', logPrefix) + await sandbox.runCommand('npm', ['pkg', 'delete', 'devDependencies.@primer/agent-eval'], { + user: NODE_USER, + }) + + logger.info('%s Installing dependencies...', logPrefix) + await sandbox.runCommand('npm', ['install'], { + user: NODE_USER, + }) + + if (trial.setup) { + logger.info('%s Running generic setup...', logPrefix) + await trial.setup({ + sandbox, + }) + } + + if (trial.treatment.setup) { + logger.info('%s Running treatment setup...', logPrefix) + await trial.treatment.setup({ + sandbox, + }) + } + + logger.info('%s Run build script...', logPrefix) + await sandbox.runCommand('npm', ['run', 'build', '--if-present'], { + user: NODE_USER, + }) + + if (trial.scenario.browserTestPath) { + logger.info('%s Installing browser test dependencies...', logPrefix) + await sandbox.runCommand( + 'npm', + ['install', '--no-save', '--package-lock=false', 'vitest', 'playwright', '@vitest/browser-playwright'], + { + user: NODE_USER, + }, + ) + logger.info('%s Installing Playwright browser...', logPrefix) + await sandbox.runCommand('./node_modules/.bin/playwright', ['install', '--with-deps', 'chromium'], { + user: 'root', + env: { + PLAYWRIGHT_BROWSERS_PATH, + }, + }) + } + + logger.info('%s Running copilot...', logPrefix) + const copilotOutput = await sandbox.runCommand( + 'copilot', + [ + '--prompt', + trial.scenario.prompt, + '--model', + trial.model.name, + '--reasoning-effort', + trial.model.reasoningEffort, + '--mode', + 'autopilot', + '--allow-all', + '--output-format', + 'json', + ], + { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + }, + ) + const messages: Array = copilotOutput.stdout.split('\n').flatMap(line => { + const trimmed = line.trim() + if (trimmed.length === 0) { + return [] + } + return parseMessage(JSON.parse(trimmed)) + }) + + logger.info('%s Running tests...', logPrefix) + + const TEST_PATH = 'scenario.test.ts' + const BROWSER_TEST_PATH = 'scenario.browser.test.ts' + const VITEST_CONFIG_PATH = 'vitest.agent-eval.config.ts' + const TEST_RESULTS_PATH = 'test-results.json' + const BROWSER_TEST_RESULTS_PATH = 'browser-test-results.json' + const scenarioTests = [ + { + sourcePath: trial.scenario.testPath, + testPath: TEST_PATH, + resultsPath: TEST_RESULTS_PATH, + browser: false, + }, + ] + + if (trial.scenario.browserTestPath) { + scenarioTests.push({ + sourcePath: trial.scenario.browserTestPath, + testPath: BROWSER_TEST_PATH, + resultsPath: BROWSER_TEST_RESULTS_PATH, + browser: true, + }) + } + + const testRuns: Array> = [] + for (const scenarioTest of scenarioTests) { + await sandbox.copy(scenarioTest.sourcePath, scenarioTest.testPath) + await sandbox.writeFile(VITEST_CONFIG_PATH, getVitestConfig(scenarioTest.resultsPath, scenarioTest.browser)) + await sandbox.runCommand( + 'sh', + ['-c', 'npx vitest run --config "$1" "$2" || true', 'vitest-run', VITEST_CONFIG_PATH, scenarioTest.testPath], + { + user: NODE_USER, + env: scenarioTest.browser ? {PLAYWRIGHT_BROWSERS_PATH} : {}, + }, + ) + + const testResultsContent = await sandbox.readFile(scenarioTest.resultsPath) + const rawTestResult: unknown = JSON.parse(testResultsContent) + const testResults = parseTestResults(rawTestResult) + if (!testResults.success) { + throw new Error(`Failed to parse test results: ${testResults.error}`) + } + + testRuns.push(testResults.data) + } + + const firstTestRun = testRuns[0] + if (!firstTestRun) { + throw new Error('No test results were collected') + } + + const testResults = + testRuns.length === 1 + ? firstTestRun + : { + ...firstTestRun, + numFailedTests: testRuns.reduce((total, result) => total + result.numFailedTests, 0), + numPassedTests: testRuns.reduce((total, result) => total + result.numPassedTests, 0), + numPendingTests: testRuns.reduce((total, result) => total + result.numPendingTests, 0), + numTodoTests: testRuns.reduce((total, result) => total + result.numTodoTests, 0), + numTotalTests: testRuns.reduce((total, result) => total + result.numTotalTests, 0), + success: testRuns.every(result => result.success), + testResults: testRuns.flatMap(result => result.testResults), + } + + if (testRuns.length > 1) { + await sandbox.writeFile(TEST_RESULTS_PATH, JSON.stringify(testResults)) + } + + const WALKTHROUGH_DIR = 'walkthrough' + const WALKTHROUGH_VIEWPORT_WIDTH = 1440 + const WALKTHROUGH_VIEWPORT_HEIGHT = 900 + logger.debug('%s Capturing walkthrough...', logPrefix) + await sandbox.runCommand('apt-get', ['install', '-y', 'chromium'], { + user: 'root', + }) + await sandbox.runCommand('npm', ['install', '-g', '--allow-scripts=agent-browser', 'agent-browser'], { + user: NODE_USER, + }) + await sandbox.runCommand( + 'npx', + ['skills', 'add', 'vercel-labs/agent-browser', '--yes', '--skill', '*', '--global', '--agent', 'github-copilot'], + { + user: NODE_USER, + }, + ) + await sandbox.writeFile( + 'agent-browser.json', + JSON.stringify({ + executablePath: '/usr/bin/chromium', + }), + ) + const walkthroughPrompt = `Record a visual walkthrough of what you implemented so a reviewer can see it without running the code themselves. + +Figure out how to start this project's server (for example by checking package.json scripts or the README) and run it in the background. Use the agent-browser CLI (already installed) to open the running app and set the browser viewport to ${WALKTHROUGH_VIEWPORT_WIDTH}x${WALKTHROUGH_VIEWPORT_HEIGHT} before capturing anything. + +Save the result inside a "${WALKTHROUGH_DIR}" directory (create it if it doesn't exist) at the root of the project: + +- If what you built is a single screen, take one screenshot and save it as ${WALKTHROUGH_DIR}/screenshot.png. +- If there are a few distinct views worth showing (for example separate pages or states), take a screenshot of each, in the order a reviewer should look at them, saved as ${WALKTHROUGH_DIR}/screenshots/01.png, ${WALKTHROUGH_DIR}/screenshots/02.png, etc. +- If reviewing the change requires seeing an interactive flow across multiple steps or pages, record a short video of yourself clicking through it instead and save it as ${WALKTHROUGH_DIR}/walkthrough.webm. + +Only capture the walkthrough, do not make any further code changes.` + const walkthroughResult = await sandbox.runCommand( + 'copilot', + [ + '--prompt', + walkthroughPrompt, + '--model', + 'gpt-5.6-terra', + '--reasoning-effort', + 'medium', + '--mode', + 'autopilot', + '--allow-all', + '--output-format', + 'json', + ], + { + user: NODE_USER, + env: { + COPILOT_GITHUB_TOKEN: copilotToken, + }, + allowNonZeroExitCode: true, + }, + ) + + if (walkthroughResult.exitCode !== 0) { + logger.warn('%s Unable to capture walkthrough: %s', logPrefix, walkthroughResult.stderr) + } + + logger.debug('%s Removing walkthrough skill...', logPrefix) + await sandbox.runCommand('rm', ['-rf', AGENT_BROWSER_SKILL_DIRECTORY], { + user: NODE_USER, + }) + + const artifactDirectory = path.join(artifactsDirectory, trial.id) + const workspaceDirectory = path.join(artifactDirectory, 'workspace') + const walkthroughPath = path.join(artifactDirectory, 'walkthrough') + const copilotConfigDirectory = path.join(artifactDirectory, '.copilot') + const skillsConfigDirectory = path.join(artifactDirectory, '.agents') + const testResultsPath = path.join(workspaceDirectory, 'test-results.json') + + if (host.existsSync(artifactDirectory)) { + await host.fs.rm(artifactDirectory, {recursive: true, force: true}) + } + await host.fs.mkdir(workspaceDirectory, {recursive: true}) + + logger.info('%s Downloading artifacts to: %s...', logPrefix, artifactDirectory) + + logger.debug('%s Downloading agent workspace to: %s...', logPrefix, workspaceDirectory) + await sandbox.download(CONTAINER_WORKDIR, workspaceDirectory, { + ignore(name) { + return name.includes('node_modules') || name.includes('.next') || name.includes('.turbo') || name.includes('dist') + }, + }) + + logger.debug('%s Downloading copilot config to: %s...', logPrefix, copilotConfigDirectory) + await sandbox.download(COPILOT_DIR, copilotConfigDirectory) + + logger.debug('%s Downloading skills config to: %s...', logPrefix, skillsConfigDirectory) + await sandbox.download(AGENTS_DIR, skillsConfigDirectory) + + let walkthrough: Walkthrough = { + type: 'Unavailable', + } + + if (host.existsSync(path.join(workspaceDirectory, WALKTHROUGH_DIR))) { + logger.debug( + '%s Moving walkthrough artifacts from: %s to: %s...', + logPrefix, + path.join(workspaceDirectory, WALKTHROUGH_DIR), + walkthroughPath, + ) + await host.fs.mkdir(walkthroughPath, {recursive: true}) + await host.fs.rename(path.join(workspaceDirectory, WALKTHROUGH_DIR), walkthroughPath) + + if (host.existsSync(path.join(walkthroughPath, 'screenshot.png'))) { + walkthrough = { + type: 'Screenshot', + filepath: path.join(walkthroughPath, 'screenshot.png'), + } + } else if (host.existsSync(path.join(walkthroughPath, 'walkthrough.webm'))) { + walkthrough = { + type: 'Video', + filepath: path.join(walkthroughPath, 'walkthrough.webm'), + } + } else if (host.existsSync(path.join(walkthroughPath, 'screenshots'))) { + const screenshotsDir = path.join(walkthroughPath, 'screenshots') + const entries = await host.fs.readdir(screenshotsDir).then(filenames => { + return filenames.toSorted((a, b) => a.localeCompare(b, undefined, {numeric: true})) + }) + const screenshots = entries.filter(entry => { + return IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase()) + }) + if (screenshots.length > 0) { + walkthrough = { + type: 'Screenshots', + screenshots: screenshots.map(screenshot => path.join(screenshotsDir, screenshot)), + } + } + } + } + + return { + artifacts: { + directory: artifactDirectory, + copilotConfigDirectory, + skillsConfigDirectory, + testResultsPath, + workspaceDirectory, + }, + trial, + agent: { + sessions: [getAgentSession(messages)], + }, + testResults, + walkthrough, + } +} + +function getAgentSession(messages: Array): AgentSession { + const turns = new Set() + const toolCalls = new Map() + let assistantOutputTokens = 0 + let modelOutputTokens = 0 + let hasModelOutput = false + + for (const message of messages) { + if (isMessageType(message, 'assistant.turn_start')) { + turns.add(message.data.turnId) + } + + if (isMessageType(message, 'assistant.message')) { + assistantOutputTokens += message.data.outputTokens ?? 0 + } + + if (isMessageType(message, 'model.message') && message.data.message.role === 'assistant') { + hasModelOutput = true + modelOutputTokens += message.data.message.outputTokens ?? 0 + } + + if (isMessageType(message, 'tool.execution_start')) { + const toolName = message.data.toolName + toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1) + } + } + + const result = messages.find(message => isMessageType(message, 'result')) + if (!result) { + throw new Error('No result message found in copilot output') + } + + return { + messages, + outputTokens: hasModelOutput ? modelOutputTokens : assistantOutputTokens, + premiumRequests: result.usage.premiumRequests, + sessionDurationMs: result.usage.sessionDurationMs, + tools: Object.fromEntries(toolCalls), + totalApiDurationMs: result.usage.totalApiDurationMs, + turns: turns.size, + } +} + +function getVitestConfig(outputFile: string, browser = false) { + const browserImport = browser ? `import {playwright} from '@vitest/browser-playwright';\n` : '' + const browserConfig = browser + ? ` browser: { + enabled: true, + headless: true, + instances: [ + { + browser: 'chromium', + }, + ], + provider: playwright(), + }, +` + : '' + + return `${browserImport}import {defineConfig} from 'vitest/config'; + +export default defineConfig({ + test: { +${browserConfig} include: ['**/*.test.ts'], + reporters: [ + [ + 'json', + { + outputFile: ${JSON.stringify(outputFile)}, + includeTaskLocation: true, + }, + ], + ], + }, +})` +} + +type CompareTrialResult = Pick + +/** + * Compare to trial results to determine which treatment performed better. We + * compare trials based on: + * + * - Test success rate (higher is better) + * - Output tokens (lower is better) + * - Total API duration (lower is better) + * - Number of turns (lower is better) + * - Number of premium requests (lower is better) + */ +function compare(a: CompareTrialResult, b: CompareTrialResult): number { + const successRateA = a.testResults.numTotalTests > 0 ? a.testResults.numPassedTests / a.testResults.numTotalTests : 0 + const successRateB = b.testResults.numTotalTests > 0 ? b.testResults.numPassedTests / b.testResults.numTotalTests : 0 + + const outputTokensA = a.agent.sessions.reduce((sum, session) => sum + session.outputTokens, 0) + const outputTokensB = b.agent.sessions.reduce((sum, session) => sum + session.outputTokens, 0) + + const totalApiDurationA = a.agent.sessions.reduce((sum, session) => sum + session.totalApiDurationMs, 0) + const totalApiDurationB = b.agent.sessions.reduce((sum, session) => sum + session.totalApiDurationMs, 0) + + const turnsA = a.agent.sessions.reduce((sum, session) => sum + session.turns, 0) + const turnsB = b.agent.sessions.reduce((sum, session) => sum + session.turns, 0) + + const premiumRequestsA = a.agent.sessions.reduce((sum, session) => sum + session.premiumRequests, 0) + const premiumRequestsB = b.agent.sessions.reduce((sum, session) => sum + session.premiumRequests, 0) + + return ( + successRateB - successRateA || + outputTokensA - outputTokensB || + totalApiDurationA - totalApiDurationB || + turnsA - turnsB || + premiumRequestsA - premiumRequestsB + ) +} + +export { + TrialSchema, + TrialResultSchema, + TrialArtifactsSchema, + TrialAgentSchema, + WalkthroughSchema, + run, + compare, + getPortableTrialPaths, +} +export type {Trial, TrialResult} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88412498..771973c5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,6 +54,18 @@ importers: specifier: ^4.1.11 version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + benchmarks: + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../packages/agent-eval + '@types/node': + specifier: ^26.1.2 + version: 26.4.0 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + experiments: devDependencies: '@primer/agent-eval': @@ -71,6 +83,15 @@ importers: dockerode: specifier: ^5.0.1 version: 5.0.1 + p-queue: + specifier: ^9.3.3 + version: 9.3.3 + pino: + specifier: ^10.3.1 + version: 10.3.1 + pino-pretty: + specifier: ^13.1.3 + version: 13.1.3 tar-fs: specifier: ^3.1.2 version: 3.1.3 @@ -93,6 +114,9 @@ importers: '@types/tar-stream': specifier: ^3.1.4 version: 3.1.4 + memfs: + specifier: ^4.68.1 + version: 4.68.1 publint: specifier: ^0.3.24 version: 0.3.24 @@ -791,6 +815,126 @@ packages: '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + '@jsonjoy.com/base64@1.1.2': + resolution: {integrity: sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/base64@17.67.0': + resolution: {integrity: sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/buffers@1.2.1': + resolution: {integrity: sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/buffers@17.67.0': + resolution: {integrity: sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/codegen@1.0.0': + resolution: {integrity: sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/codegen@17.67.0': + resolution: {integrity: sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-core@4.68.1': + resolution: {integrity: sha512-V5oZ4Gt9WJKyQef0n9cAd0N9qjSkIBm3E4MYsgNIWBk5aINCDPKxMPo1i29rBxqiT4Ixf1epklqV9VJMKIxwlw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-fsa@4.68.1': + resolution: {integrity: sha512-HCG72UioncuO7Gw09XNVG+S85e3cq2hrUC/mexBrsWsa3mI7eePkkqWie3uVYbtsb64OR9YGQs5SqaufDRYBcg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-builtins@4.68.1': + resolution: {integrity: sha512-HK1BTksysokNZxNspqDH0yPaqN9YgR/AYIlYiIaU2Ys4BOk5CdybI7r6BgiZuiiPiV8n4sK/kZdice7Znpy2Kw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-to-fsa@4.68.1': + resolution: {integrity: sha512-lpKmU4X9e/oh8GIuAI7EXaS5QiLNM3KD15CkdhfS6PYmrGvoJqKQcyEfnLgnnaGslh/PFUMYSIZBCf2ejJGw8g==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-utils@4.68.1': + resolution: {integrity: sha512-/GxfW1DWm9SCdkfbvqevLO/P5duobQfmKkHXxdMIDbcZMQeAgooAstIfZhkXpATzq9QbCQsnoWFM/dGHdZfndw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node@4.68.1': + resolution: {integrity: sha512-R5D9mWtqdURzcOWj1vdXr3APCwX0xchtFT+kmW7fXLNDifWdDrnh26jSID8pdnUfFBxTyfHtFtTL/NWKzIH7kQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-print@4.68.1': + resolution: {integrity: sha512-oGeZOGPYKK9v1CgeVeEDsLomH1lCnslSpqUN5GmPzrmAVGQlsmsdcXNA2O4lV8Y4xkuSuynx2ITBkUHJVaTbow==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-snapshot@4.68.1': + resolution: {integrity: sha512-XZfP0FDZN32bbc4t2bZN2qRrYHg5AktJnzk22HRoKGK4BprrbNRH2k5ceSNS/kupKYcofCs+O841+xaAbjnxwQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pack@1.21.0': + resolution: {integrity: sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pack@17.67.0': + resolution: {integrity: sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pointer@1.0.2': + resolution: {integrity: sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pointer@17.67.0': + resolution: {integrity: sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/util@1.9.0': + resolution: {integrity: sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/util@17.67.0': + resolution: {integrity: sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + '@lit-labs/react@1.2.1': resolution: {integrity: sha512-DiZdJYFU0tBbdQkfwwRSwYyI/mcWkg3sWesKRsHUd4G+NekTmmeq9fzsurvcKTNVa0comNljwtg4Hvi1ds3V+A==} @@ -870,6 +1014,9 @@ packages: '@oxc-project/types@0.147.0': resolution: {integrity: sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==} + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@pkgr/core@0.3.6': resolution: {integrity: sha512-SEeaJLb3qBNF/OaXnaR1NmmBbFYk1zC0ZH/52fATcRPLFg/p791YrcyFFy44Bo9sLaGuSuLp5Q6axbb/O+v/RA==} engines: {node: ^14.18.0 || >=16.0.0} @@ -1584,6 +1731,10 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + available-typed-arrays@1.0.7: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} @@ -1743,6 +1894,9 @@ packages: color2k@2.0.4: resolution: {integrity: sha512-OXAPGFRNeLFnUfqDtloYdxkwsJoIdXe28+bjbpJiPqyei2HPa3VHmMCWa0Qe62+U4Ftf9Hj7hRssOkxz7WiWbg==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -1775,6 +1929,9 @@ packages: resolution: {integrity: sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==} engines: {node: '>= 0.4'} + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -2047,6 +2204,9 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + eventemitter3@5.0.4: + resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + events-universal@1.0.1: resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} @@ -2054,6 +2214,9 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fast-copy@4.0.4: + resolution: {integrity: sha512-eVAiWVNPSEGIzDl5yPuLrx8fNMogScXvD9xp1Kzd41FjRIz2I3sSIcxsFeM5EzFfHAfobdvs8ZySffUopljvIA==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2069,6 +2232,9 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -2160,6 +2326,12 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} + glob-to-regex.js@1.2.0: + resolution: {integrity: sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + glob@13.0.6: resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} engines: {node: 18 || 20 || >=22} @@ -2206,6 +2378,9 @@ packages: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + history@5.3.0: resolution: {integrity: sha512-ZqaKwjjrAYUYfLG+htGaIIZ4nioX2L70ZUMIFysS3xvBsSG4x/n1V6TXV3N8ZYNuFGlDirFg32T7B6WOUPDYcQ==} @@ -2216,6 +2391,10 @@ packages: resolution: {integrity: sha512-K3GbkIWqyvvlpfhBPlbEvD97TtqBpAYA4kt+cn2lD2x2HuohzZCibcA2nOlnJT6exqvJLggoB5nv2dNf192nEA==} hasBin: true + hyperdyperid@1.2.0: + resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} + engines: {node: '>=10.18'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -2362,6 +2541,10 @@ packages: jju@1.4.0: resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==} + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + js-yaml@4.3.0: resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} hasBin: true @@ -2587,6 +2770,9 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + memfs@4.68.1: + resolution: {integrity: sha512-OD+IDRUvIxu3QHL+nFm9gdyugInD27FDJ+sl4B5QgomPHXMlbw+GP918P8VNKu2FkNlVeqBkpzkwROpamVifRw==} + minimatch@10.2.6: resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} engines: {node: 18 || 20 || >=22} @@ -2688,6 +2874,10 @@ packages: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -2707,6 +2897,14 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-queue@9.3.3: + resolution: {integrity: sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==} + engines: {node: '>=20'} + + p-timeout@7.0.1: + resolution: {integrity: sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==} + engines: {node: '>=20'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -2742,6 +2940,20 @@ packages: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-pretty@13.1.3: + resolution: {integrity: sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==} + hasBin: true + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@10.3.1: + resolution: {integrity: sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==} + hasBin: true + playwright-core@1.62.1: resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} engines: {node: '>=20'} @@ -2782,6 +2994,9 @@ packages: engines: {node: '>=14'} hasBin: true + process-warning@5.1.0: + resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==} + protobufjs@7.6.5: resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} engines: {node: '>=12.0.0'} @@ -2798,6 +3013,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + react-compiler-runtime@1.0.0: resolution: {integrity: sha512-rRfjYv66HlG8896yPUDONgKzG5BxZD1nV9U6rkm+7VCuvQc903C4MjcoZR4zPw53IKSOX9wMQVpA1IAbRtzQ7w==} peerDependencies: @@ -2828,6 +3046,13 @@ packages: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + real-require@1.0.0: + resolution: {integrity: sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==} + reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -2900,12 +3125,19 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -2970,6 +3202,9 @@ packages: sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -2977,6 +3212,10 @@ packages: split-ca@1.0.1: resolution: {integrity: sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==} + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + ssh2@1.17.0: resolution: {integrity: sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==} engines: {node: '>=10.16.0'} @@ -3029,6 +3268,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strip-json-comments@5.0.3: + resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} + engines: {node: '>=14.16'} + styled-jsx@5.1.6: resolution: {integrity: sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==} engines: {node: '>= 12.0.0'} @@ -3079,6 +3322,16 @@ packages: text-decoder@1.2.7: resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + thingies@2.6.1: + resolution: {integrity: sha512-cV/CMGTK3M4MlnJ/0At6ismOw/A0EEniDNScajjz/Br3c1sqE72YD01rGpPTKwd27wAxI5Pr+6+0w8yofzFRYw==} + engines: {node: '>=10.18'} + peerDependencies: + tslib: ^2 + + thread-stream@4.2.0: + resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} + engines: {node: '>=20'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -3094,6 +3347,12 @@ packages: resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} + tree-dump@1.1.0: + resolution: {integrity: sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -3688,6 +3947,134 @@ snapshots: '@js-sdsl/ordered-map@4.4.2': {} + '@jsonjoy.com/base64@1.1.2(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/base64@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/buffers@1.2.1(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/buffers@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/codegen@1.0.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/codegen@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/fs-core@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-fsa@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-builtins@4.68.1(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-to-fsa@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-fsa': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-utils@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + glob-to-regex.js: 1.2.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-print': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-snapshot': 4.68.1(tslib@2.8.1) + glob-to-regex.js: 1.2.0(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-print@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-snapshot@4.68.1(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/json-pack': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pack@1.21.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/base64': 1.1.2(tslib@2.8.1) + '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1) + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + '@jsonjoy.com/json-pointer': 1.0.2(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + hyperdyperid: 1.2.0 + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pack@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/base64': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/json-pointer': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + hyperdyperid: 1.2.0 + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pointer@1.0.2(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pointer@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/util@1.9.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1) + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/util@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + '@lit-labs/react@1.2.1': {} '@lit-labs/ssr-dom-shim@1.6.0': {} @@ -3737,6 +4124,8 @@ snapshots: '@oxc-project/types@0.147.0': {} + '@pinojs/redact@0.4.0': {} + '@pkgr/core@0.3.6': {} '@pnpm/deps.graph-sequencer@1100.0.1': {} @@ -4397,6 +4786,8 @@ snapshots: async-function@1.0.0: {} + atomic-sleep@1.0.0: {} + available-typed-arrays@1.0.7: dependencies: possible-typed-array-names: 1.1.0 @@ -4532,6 +4923,8 @@ snapshots: color2k@2.0.4: {} + colorette@2.0.20: {} + concat-map@0.0.1: {} convert-source-map@2.0.0: {} @@ -4570,6 +4963,8 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.2 + dateformat@4.6.3: {} + debug@3.2.7: dependencies: ms: 2.1.3 @@ -4959,6 +5354,8 @@ snapshots: esutils@2.0.3: {} + eventemitter3@5.0.4: {} + events-universal@1.0.1: dependencies: bare-events: 2.9.1 @@ -4967,6 +5364,8 @@ snapshots: expect-type@1.4.0: {} + fast-copy@4.0.4: {} + fast-deep-equal@3.1.3: {} fast-diff@1.3.0: {} @@ -4977,6 +5376,8 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-safe-stringify@2.1.1: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -5073,6 +5474,10 @@ snapshots: dependencies: is-glob: 4.0.3 + glob-to-regex.js@1.2.0(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + glob@13.0.6: dependencies: minimatch: 10.2.6 @@ -5112,6 +5517,8 @@ snapshots: dependencies: function-bind: 1.1.2 + help-me@5.0.0: {} + history@5.3.0: dependencies: '@babel/runtime': 7.29.7 @@ -5120,6 +5527,8 @@ snapshots: human-id@4.2.0: {} + hyperdyperid@1.2.0: {} + ieee754@1.2.1: {} ignore@5.3.2: {} @@ -5263,6 +5672,8 @@ snapshots: jju@1.4.0: {} + joycon@3.1.1: {} + js-yaml@4.3.0: dependencies: argparse: 2.0.1 @@ -5430,6 +5841,23 @@ snapshots: math-intrinsics@1.1.0: {} + memfs@4.68.1: + dependencies: + '@jsonjoy.com/fs-core': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-fsa': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-to-fsa': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-print': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/fs-snapshot': 4.68.1(tslib@2.8.1) + '@jsonjoy.com/json-pack': 1.21.0(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + glob-to-regex.js: 1.2.0(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + minimatch@10.2.6: dependencies: brace-expansion: 5.0.9 @@ -5534,6 +5962,8 @@ snapshots: obug@2.1.4: {} + on-exit-leak-free@2.1.2: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -5562,6 +5992,13 @@ snapshots: dependencies: p-limit: 3.1.0 + p-queue@9.3.3: + dependencies: + eventemitter3: 5.0.4 + p-timeout: 7.0.1 + + p-timeout@7.0.1: {} + package-json-from-dist@1.0.1: {} package-manager-detector@1.8.0: {} @@ -5587,6 +6024,42 @@ snapshots: picomatch@4.0.5: {} + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-pretty@13.1.3: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 4.0.4 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pump: 3.0.4 + secure-json-parse: 4.1.0 + sonic-boom: 4.2.1 + strip-json-comments: 5.0.3 + + pino-std-serializers@7.1.0: {} + + pino@10.3.1: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.1.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 4.2.0 + playwright-core@1.62.1: {} playwright@1.62.1: @@ -5619,6 +6092,8 @@ snapshots: prettier@3.9.6: {} + process-warning@5.1.0: {} + protobufjs@7.6.5: dependencies: '@protobufjs/aspromise': 1.1.2 @@ -5647,6 +6122,8 @@ snapshots: punycode@2.3.1: {} + quick-format-unescaped@4.0.4: {} + react-compiler-runtime@1.0.0(react@19.2.8): dependencies: react: 19.2.8 @@ -5672,6 +6149,10 @@ snapshots: string_decoder: 1.3.0 util-deprecate: 1.0.2 + real-require@0.2.0: {} + + real-require@1.0.0: {} + reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -5772,10 +6253,14 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safe-stable-stringify@2.5.0: {} + safer-buffer@2.1.2: {} scheduler@0.27.0: {} + secure-json-parse@4.1.0: {} + semver@6.3.1: {} semver@7.8.5: {} @@ -5876,10 +6361,16 @@ snapshots: sisteransi@1.0.5: {} + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + source-map-js@1.2.1: {} split-ca@1.0.1: {} + split2@4.2.0: {} + ssh2@1.17.0: dependencies: asn1: 0.2.6 @@ -5954,6 +6445,8 @@ snapshots: strip-json-comments@3.1.1: {} + strip-json-comments@5.0.3: {} + styled-jsx@5.1.6(react@19.2.8): dependencies: client-only: 0.0.1 @@ -6022,6 +6515,14 @@ snapshots: transitivePeerDependencies: - react-native-b4a + thingies@2.6.1(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + + thread-stream@4.2.0: + dependencies: + real-require: 1.0.0 + tinybench@2.9.0: {} tinyexec@1.3.0: {} @@ -6033,6 +6534,10 @@ snapshots: tinyrainbow@3.1.0: {} + tree-dump@1.1.0(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + ts-api-utils@2.5.0(typescript@6.0.3): dependencies: typescript: 6.0.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 261cda7d..9f0e4be8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,7 +7,8 @@ allowBuilds: minimumReleaseAge: 10080 packages: - - 'scenarios/*' + - 'benchmarks' - 'experiments' - 'packages/*' + - 'scenarios/*' - 'website' diff --git a/scenarios/000-nextjs-template/scenario.config.ts b/scenarios/000-nextjs-template/scenario.config.ts index 89ab54e3..d153a49d 100644 --- a/scenarios/000-nextjs-template/scenario.config.ts +++ b/scenarios/000-nextjs-template/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ prompt: `Example prompt`, tags: [], }) diff --git a/scenarios/000-vite-template/scenario.config.ts b/scenarios/000-vite-template/scenario.config.ts index 02803bff..9c437b85 100644 --- a/scenarios/000-vite-template/scenario.config.ts +++ b/scenarios/000-vite-template/scenario.config.ts @@ -1,5 +1,5 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ prompt: `Example prompt`, }) diff --git a/scenarios/001-agent-uses-button-from-primer/scenario.config.ts b/scenarios/001-agent-uses-button-from-primer/scenario.config.ts index b3e98827..67f0a0d6 100644 --- a/scenarios/001-agent-uses-button-from-primer/scenario.config.ts +++ b/scenarios/001-agent-uses-button-from-primer/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent uses a primary Primer button with the requested label.', prompt: `Update the index page to use a primary button with the text 'Submit'`, tags: ['baseline', 'button', 'component', 'nextjs', 'primer'], diff --git a/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts b/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts index f4806c80..98885963 100644 --- a/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts +++ b/scenarios/002-agent-uses-octicon-from-primer/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent uses the requested icon from Primer Octicons.', prompt: `Update the index page to use a Search icon`, tags: ['baseline', 'icon', 'nextjs', 'primer'], diff --git a/scenarios/003-agent-uses-form-from-primer/scenario.config.ts b/scenarios/003-agent-uses-form-from-primer/scenario.config.ts index 0fe02048..eede7936 100644 --- a/scenarios/003-agent-uses-form-from-primer/scenario.config.ts +++ b/scenarios/003-agent-uses-form-from-primer/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent builds a sign-up form with Primer form components.', prompt: `Update the index page to render a sign-up form. The form does not need to post to an endpoint, I am only working on the UI for now.`, tags: ['baseline', 'component', 'form', 'nextjs', 'primer'], diff --git a/scenarios/004-agent-setup-nextjs/scenario.config.ts b/scenarios/004-agent-setup-nextjs/scenario.config.ts index c74cebe4..cd855be8 100644 --- a/scenarios/004-agent-setup-nextjs/scenario.config.ts +++ b/scenarios/004-agent-setup-nextjs/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent configures a Next.js project to use Primer.', prompt: `Setup this project to work with Primer. Including a default page layout in app.tsx.`, tags: ['baseline', 'nextjs', 'primer', 'setup'], diff --git a/scenarios/005-agent-enables-theme-switching/scenario.config.ts b/scenarios/005-agent-enables-theme-switching/scenario.config.ts index 7eab4360..b01f5e6a 100644 --- a/scenarios/005-agent-enables-theme-switching/scenario.config.ts +++ b/scenarios/005-agent-enables-theme-switching/scenario.config.ts @@ -1,6 +1,6 @@ -import {defineScenario} from '@primer/agent-eval/scenario' +import {defineConfig} from '@primer/agent-eval/scenario' -export default defineScenario({ +export default defineConfig({ description: 'Evaluate whether the agent enables every available Primer color scheme.', prompt: `Enable support for switching between all available color schemes.`, tags: ['accessibility', 'baseline', 'nextjs', 'primer', 'theming'], diff --git a/script/run-baseline.sh b/script/run-baseline.sh deleted file mode 100755 index acc4f35e..00000000 --- a/script/run-baseline.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -run_date="${RUN_DATE:-$(date -u +%F)}" -output_file="${OUTPUT_FILE:-output.json}" - -if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then - echo "RUN_DATE must use the YYYY-MM-DD format" >&2 - exit 1 -fi - -arguments=( - --artifacts "$repository_root/artifacts" - --concurrency "${CONCURRENCY:-1}" - --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" - --experiment baseline - --experiments "$repository_root/experiments" - --output "$repository_root/results/$run_date/$output_file" - --scenarios "$repository_root/scenarios" -) - -if [[ -n "${SHARD:-}" ]]; then - arguments+=(--shard "$SHARD") -fi - -node "$repository_root/packages/agent-eval/dist/cli.js" "${arguments[@]}" diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh new file mode 100755 index 00000000..687d7486 --- /dev/null +++ b/script/run-benchmark.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +benchmark_name="${BENCHMARK_NAME:-design-system}" +run_date="${RUN_DATE:-$(date -u +%F)}" +run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date" + +if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "BENCHMARK_NAME must be a benchmark file name without its extension" >&2 + exit 1 +fi + +if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "RUN_DATE must use the YYYY-MM-DD format" >&2 + exit 1 +fi + +node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmark "$benchmark_name" \ + --benchmarks "$repository_root/benchmarks" \ + --concurrency "${CONCURRENCY:-1}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" diff --git a/script/setup b/script/setup index 48f14108..2305a4fd 100755 --- a/script/setup +++ b/script/setup @@ -9,10 +9,7 @@ corepack install # Install dependencies pnpm install --frozen-lockfile -# Baseline workspaces are not needed after their merged results are produced. -rm -rf "$repository_root/artifacts" - -# Download results from the 10 most recent retained baseline artifacts +# Download the 10 most recent retained benchmark result bundles. artifact_ids="$( gh api \ --paginate \ @@ -20,7 +17,7 @@ artifact_ids="$( "/repos/primer/agent-eval/actions/artifacts?per_page=100" | jq --raw-output ' [.[].artifacts[] - | select(.expired == false and (.name | test("^baseline-[0-9]+$"))) + | select(.expired == false and (.name | test("^benchmark-[0-9]+$"))) ] | sort_by(.created_at) | .[-10:][].id diff --git a/website/README.md b/website/README.md index 0d1824b8..2ad28f84 100644 --- a/website/README.md +++ b/website/README.md @@ -2,10 +2,31 @@ ## Routes -| URL | Description | -| :----------------- | :--------------------------------------------------------------------------- | -| `/` | View baseline above the fold, section for experiments, section for scenarios | -| `/experiments` | List experiments | -| `/experiments/:id` | View experiment details | -| `/scenarios` | List scenarios | -| `/scenarios/:id` | View scenario details and baseline results | +| URL | Description | +| :---------------------------- | :---------------------------------------------- | +| `/` | View the latest design system benchmark results | +| `/benchmarks` | List benchmarks | +| `/benchmarks/:id` | View benchmark results and dated runs | +| `/benchmarks/:id/runs/:date` | View benchmark run details and walkthroughs | +| `/experiments` | List experiments | +| `/experiments/:id` | View experiment details | +| `/experiments/:id/runs/:date` | View experiment run details and walkthroughs | +| `/scenarios` | List scenarios | +| `/scenarios/:id` | View scenario details | + +## Results + +The website reads portable result bundles from: + +```text +results/ +├── benchmarks/// +│ ├── output.json +│ └── artifacts/ +└── experiments/// + ├── output.json + └── artifacts/ +``` + +Artifact and walkthrough paths are relative to each `output.json`, so result +directories should be moved or uploaded as complete bundles. diff --git a/website/src/app/baseline/[scenarioId]/page.tsx b/website/src/app/baseline/[scenarioId]/page.tsx deleted file mode 100644 index fbbe0d33..00000000 --- a/website/src/app/baseline/[scenarioId]/page.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import {notFound} from 'next/navigation' -import {getBaselinePageData} from '../../../baseline-results' -import {list as listScenarios} from '../../../scenarios' -import {Index} from '../../components/Index' - -export const dynamicParams = false - -type BaselineScenarioPageProps = { - params: Promise<{ - scenarioId: string - }> -} - -export default async function BaselineScenarioPage({params}: BaselineScenarioPageProps) { - const {scenarioId} = await params - const [{baseline, baselineTrends}, scenarios] = await Promise.all([getBaselinePageData(), listScenarios()]) - - if (!scenarios.some(scenario => scenario.id === scenarioId)) { - notFound() - } - - return -} - -export async function generateStaticParams() { - const scenarios = await listScenarios() - return scenarios.map(scenario => ({scenarioId: scenario.id})) -} diff --git a/website/src/app/baseline/page.tsx b/website/src/app/baseline/page.tsx deleted file mode 100644 index 79d4b3ea..00000000 --- a/website/src/app/baseline/page.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import {getBaselinePageData} from '../../baseline-results' -import {Index} from '../components/Index' - -export default async function BaselinePage() { - const {baseline, baselineTrends} = await getBaselinePageData() - return -} diff --git a/website/src/app/benchmarks/[id]/components/Page.tsx b/website/src/app/benchmarks/[id]/components/Page.tsx new file mode 100644 index 00000000..5fefaa57 --- /dev/null +++ b/website/src/app/benchmarks/[id]/components/Page.tsx @@ -0,0 +1,233 @@ +'use client' + +import {Breadcrumbs, Stack} from '@primer/react' +import {Blankslate, DataTable, Table} from '@primer/react/experimental' +import type {Route} from 'next' +import NextLink from 'next/link' +import type {BenchmarkPageResults} from '../../../../benchmark-results' +import type {Benchmark} from '../../../../benchmarks' +import {Link} from '../../../../components/Link' + +type Comparison = BenchmarkPageResults['capabilities'][number]['comparison'] + +type TableResult = { + id: string + capability: string + scenario: string + scenarioId?: string + model: string + reasoningEffort: string + comparison: Comparison +} + +type BenchmarkRun = { + id: string + name: string + resultCount: number + passedTests: number + totalTests: number +} + +function createTableResults(results: BenchmarkPageResults): Array { + return results.capabilities.flatMap(capability => { + return [ + { + id: capability.id, + capability: capability.name, + scenario: capability.scenarios.length > 0 ? 'All scenarios' : 'Not measured', + model: 'All models', + reasoningEffort: '', + comparison: capability.comparison, + }, + ...capability.scenarios.flatMap(scenario => { + return [ + { + id: `${capability.id}-${scenario.id}`, + capability: '', + scenario: scenario.id, + scenarioId: scenario.id, + model: 'All models', + reasoningEffort: '', + comparison: scenario.comparison, + }, + ...scenario.models.map(model => { + return { + id: `${capability.id}-${scenario.id}-${model.id}`, + capability: '', + scenario: '', + model: model.name, + reasoningEffort: model.reasoningEffort, + comparison: model.comparison, + } + }), + ] + }), + ] + }) +} + +export function Page({ + benchmark, + results, + runs, +}: { + benchmark: Benchmark + results: BenchmarkPageResults | null + runs: Array +}) { + const tableResults = results ? createTableResults(results) : [] + + return ( + + + + Benchmarks + + {benchmark.id} + +
+

{benchmark.name}

+

{benchmark.description}

+
+
+

Runs

+ {runs.length > 0 ? ( + + { + return ( + + + + ) + }, + }, + {id: 'results', header: 'Results', field: 'resultCount', align: 'end'}, + { + id: 'tests', + header: 'Tests passed', + field: 'passedTests', + align: 'end', + renderCell: row => { + return `${row.passedTests}/${row.totalTests}` + }, + }, + ]} + data={runs} + /> + + ) : ( + + No runs + No results have been recorded for this benchmark yet. + + )} +
+ {results ? ( + + + Capability performance + + + Benchmark results are shown first, followed by the percent change from Control in parentheses. Capability + rows aggregate all of their scenario and model results. Latest results:{' '} + + + + . + + { + return row.scenarioId ? ( + {row.scenario} + ) : ( + row.scenario + ) + }, + }, + { + id: 'model', + header: 'Model', + field: 'model', + }, + { + id: 'reasoning-effort', + header: 'Reasoning effort', + field: 'reasoningEffort', + }, + { + id: 'tests', + header: 'Tests', + field: 'comparison', + align: 'end', + renderCell: row => { + return row.comparison.tests + }, + }, + { + id: 'output-tokens', + header: 'Output tokens', + field: 'comparison', + align: 'end', + renderCell: row => { + return row.comparison.outputTokens + }, + }, + { + id: 'premium-requests', + header: 'Premium requests', + field: 'comparison', + align: 'end', + renderCell: row => { + return row.comparison.premiumRequests + }, + }, + { + id: 'session-time', + header: 'Session time', + field: 'comparison', + align: 'end', + renderCell: row => { + return row.comparison.sessionTime + }, + }, + { + id: 'api-time', + header: 'API time', + field: 'comparison', + align: 'end', + renderCell: row => { + return row.comparison.apiTime + }, + }, + ]} + data={tableResults} + /> + + ) : null} + + ) +} diff --git a/website/src/app/benchmarks/[id]/page.tsx b/website/src/app/benchmarks/[id]/page.tsx new file mode 100644 index 00000000..a0a395c9 --- /dev/null +++ b/website/src/app/benchmarks/[id]/page.tsx @@ -0,0 +1,27 @@ +import {getBenchmarkPageData} from '../../../benchmark-page-data' +import {list} from '../../../benchmarks' +import {Page} from './components/Page' + +type BenchmarkPageProps = { + params: Promise<{ + id: string + }> +} + +export const dynamicParams = false + +export default async function BenchmarkPage(props: BenchmarkPageProps) { + const {id} = await props.params + const {benchmark, results, runs} = await getBenchmarkPageData(id) + + return +} + +export async function generateStaticParams() { + const benchmarks = await list() + return benchmarks.map(benchmark => { + return { + id: benchmark.id, + } + }) +} diff --git a/website/src/app/benchmarks/[id]/runs/[date]/page.tsx b/website/src/app/benchmarks/[id]/runs/[date]/page.tsx new file mode 100644 index 00000000..4ed36847 --- /dev/null +++ b/website/src/app/benchmarks/[id]/runs/[date]/page.tsx @@ -0,0 +1,121 @@ +import type {BenchmarkOutput} from '@primer/agent-eval/benchmark' +import type {Route} from 'next' +import {notFound} from 'next/navigation' +import {get as getBenchmark, list as listBenchmarks} from '../../../../../benchmarks' +import {getBenchmarkRun, listBenchmarkRuns, type BenchmarkRun} from '../../../../../benchmark-results' +import {createTranscript, getWalkthroughDataUrls, type RunDetails} from '../../../../../run-details' +import {RunDetailsPage} from '../../../../components/RunDetailsPage' + +const EMPTY_RUN_PARAM = '__no-runs__' + +type BenchmarkOutputTrial = BenchmarkOutput['trials'] extends Map ? Trial : never + +type RunPageProps = { + params: Promise<{ + id: string + date: string + }> +} + +async function createBenchmarkRunDetails(run: BenchmarkRun): Promise { + const treatments = new Map( + [...run.output.treatments].map(([id, treatment]) => { + return [id, treatment.name] + }), + ) + + return { + date: run.name, + results: await Promise.all( + [...run.output.trials.values()].map(async (trial: BenchmarkOutputTrial) => { + const sessions = trial.agent.sessions + return { + id: trial.id, + scenarioId: trial.scenarioId, + context: trial.capabilityId, + treatment: treatments.get(trial.treatmentId) ?? 'Unknown treatment', + model: trial.model.name, + reasoningEffort: trial.model.reasoningEffort, + testsPassed: trial.testResults.numPassedTests, + totalTests: trial.testResults.numTotalTests, + turns: sessions.reduce((total, session) => { + return total + session.turns + }, 0), + outputTokens: sessions.reduce((total, session) => { + return total + session.outputTokens + }, 0), + premiumRequests: sessions.reduce((total, session) => { + return total + session.premiumRequests + }, 0), + totalApiDurationMs: sessions.reduce((total, session) => { + return total + session.totalApiDurationMs + }, 0), + sessionDurationMs: sessions.reduce((total, session) => { + return total + session.sessionDurationMs + }, 0), + tests: trial.testResults.testResults.flatMap(testResult => { + return testResult.assertionResults.map(assertion => { + return { + fullName: assertion.fullName, + status: assertion.status, + description: assertion.meta.description, + } + }) + }), + walkthrough: await getWalkthroughDataUrls(trial.walkthrough, run.directory), + transcript: createTranscript( + sessions.flatMap(session => { + return session.messages + }), + ), + } + }), + ), + } +} + +export const dynamicParams = false + +export default async function BenchmarkRunPage(props: RunPageProps) { + const {id, date} = await props.params + if (id === EMPTY_RUN_PARAM && date === EMPTY_RUN_PARAM) { + notFound() + } + + const [benchmark, run] = await Promise.all([getBenchmark(id), getBenchmarkRun(id, date)]) + if (!run) { + notFound() + } + + return ( + + ) +} + +export async function generateStaticParams() { + const benchmarks = await listBenchmarks() + const params = ( + await Promise.all( + benchmarks.map(async benchmark => { + const runs = await listBenchmarkRuns(benchmark.id) + return runs.map(run => { + return { + id: benchmark.id, + date: run.name, + } + }) + }), + ) + ).flat() + + return params.length > 0 ? params : [{id: EMPTY_RUN_PARAM, date: EMPTY_RUN_PARAM}] +} diff --git a/website/src/app/benchmarks/page.tsx b/website/src/app/benchmarks/page.tsx new file mode 100644 index 00000000..a5c0d55f --- /dev/null +++ b/website/src/app/benchmarks/page.tsx @@ -0,0 +1,11 @@ +import {BenchmarksTable} from '../components/ResourceTables' +import {list as listBenchmarks} from '../../benchmarks' + +export const metadata = { + title: 'Benchmarks', +} + +export default async function BenchmarksPage() { + const benchmarks = await listBenchmarks() + return +} diff --git a/website/src/app/components/BaselineOverview.tsx b/website/src/app/components/BaselineOverview.tsx deleted file mode 100644 index a019098e..00000000 --- a/website/src/app/components/BaselineOverview.tsx +++ /dev/null @@ -1,33 +0,0 @@ -'use client' - -import {Stack} from '@primer/react' -import type {BaselineTrendPoint} from './BaselineTrends' -import {BaselineTrends} from './BaselineTrends' -import type {BaselineResult} from './Index' -import {BaselineResultsTable} from './Index' - -export function BaselineOverview({ - results, - trends, -}: { - results: Array - trends: Array -}) { - return ( - - - Each metric is the average Recommended result across all baseline scenarios, followed by the percent change - from the average Control result in parentheses. - - } - heading="Baseline overview" - headingId="aggregate-baseline-heading" - headingLevel="h1" - results={results} - /> - - - ) -} diff --git a/website/src/app/components/BaselineTrends.tsx b/website/src/app/components/BaselineTrends.tsx deleted file mode 100644 index 5fe39887..00000000 --- a/website/src/app/components/BaselineTrends.tsx +++ /dev/null @@ -1,581 +0,0 @@ -'use client' - -import {Button, FormControl, Heading, Select, Stack, Text} from '@primer/react' -import {DataTable, Table} from '@primer/react/experimental' -import {type MouseEvent, useState} from 'react' -import styles from './BaselineTrends.module.css' - -type BaselineTrendMetric = { - value: number | null - raw: string - change: number | null - controlValue: number | null - controlRaw: string | null -} - -type BaselineTrendMetricId = - 'tests' | 'turns' | 'outputTokens' | 'premiumRequests' | 'apiDuration' | 'sessionDuration' | 'toolCalls' - -export type BaselineTrendPoint = { - id: string - date: string - scenarioId: string - model: string - reasoningEffort: string - metrics: Record -} - -const metrics: Array<{ - id: BaselineTrendMetricId - label: string -}> = [ - {id: 'tests', label: 'Tests passed'}, - {id: 'turns', label: 'Turns'}, - {id: 'outputTokens', label: 'Output tokens'}, - {id: 'premiumRequests', label: 'Premium requests'}, - {id: 'apiDuration', label: 'API time'}, - {id: 'sessionDuration', label: 'Session time'}, - {id: 'toolCalls', label: 'Tool calls'}, -] - -const lineStyles = [ - {color: 'var(--data-blue-color-emphasis, var(--data-blue-color))', dash: undefined}, - {color: 'var(--data-green-color-emphasis, var(--data-green-color))', dash: '7 3'}, - {color: 'var(--data-orange-color-emphasis, var(--data-orange-color))', dash: '2 3'}, - {color: 'var(--data-pink-color-emphasis, var(--data-pink-color))', dash: '10 3 2 3'}, - {color: 'var(--data-yellow-color-emphasis, var(--data-yellow-color))', dash: '4 4'}, - {color: 'var(--data-red-color-emphasis, var(--data-red-color))', dash: '12 4'}, -] - -const percentFormatter = new Intl.NumberFormat('en-US', { - maximumFractionDigits: 1, -}) - -const numberFormatter = new Intl.NumberFormat('en-US', { - maximumFractionDigits: 1, -}) - -function getSeriesName(point: BaselineTrendPoint) { - return point.reasoningEffort === '—' ? point.model : `${point.model} (${point.reasoningEffort})` -} - -function formatChange(change: number | null) { - if (change === null) { - return '—' - } - - return `${change > 0 ? '+' : ''}${percentFormatter.format(change)}%` -} - -function TrendChart({ - dates, - metric, - points, - series, - highlightedSeries, - highlightedTreatment, -}: { - dates: Array - metric: (typeof metrics)[number] - points: Array - series: Array - highlightedSeries: string | null - highlightedTreatment: 'recommended' | 'control' | null -}) { - const [activeTrend, setActiveTrend] = useState<{ - pointId: string - treatment: 'recommended' | 'control' - } | null>(null) - const width = 520 - const height = 240 - const padding = {top: 16, right: 16, bottom: 52, left: 48} - const plotWidth = width - padding.left - padding.right - const plotHeight = height - padding.top - padding.bottom - const values = points.flatMap(point => { - const value = point.metrics[metric.id] - return [value.value, value.controlValue].filter(item => item !== null) - }) - const rawMin = values.length > 0 ? Math.min(...values) : 0 - const rawMax = values.length > 0 ? Math.max(...values) : 0 - const range = rawMax - rawMin - const min = Math.max(0, rawMin - (range || Math.max(rawMax, 1)) * 0.1) - const max = rawMax + (range || Math.max(rawMax, 1)) * 0.1 - const getX = (date: string) => - padding.left + (dates.length === 1 ? plotWidth / 2 : (dates.indexOf(date) / (dates.length - 1)) * plotWidth) - const getY = (value: number) => padding.top + (1 - (value - min) / (max - min)) * plotHeight - const activePoint = points.find(point => point.id === activeTrend?.pointId) - const activeMetric = activePoint?.metrics[metric.id] - const activeValue = - activeTrend?.treatment === 'control' - ? (activeMetric?.controlValue ?? undefined) - : (activeMetric?.value ?? undefined) - const activeRaw = activeTrend?.treatment === 'control' ? activeMetric?.controlRaw : activeMetric?.raw - const tooltipWidth = 248 - const tooltipHeight = 52 - const tooltipX = activePoint - ? Math.min(Math.max(getX(activePoint.date) - tooltipWidth / 2, padding.left), width - padding.right - tooltipWidth) - : 0 - const tooltipY = - activePoint && activeValue !== undefined - ? Math.max(padding.top, Math.min(getY(activeValue) - tooltipHeight - 10, height - padding.bottom - tooltipHeight)) - : 0 - const tooltipId = `${metric.id}-chart-tooltip` - const controlOpacity = highlightedTreatment === 'recommended' ? 0.15 : highlightedTreatment === 'control' ? 1 : 0.4 - const recommendedOpacity = highlightedTreatment === 'control' ? 0.15 : 1 - const activateClosestPoint = ( - event: MouseEvent, - seriesPoints: Array, - treatment: 'recommended' | 'control', - ) => { - const svg = event.currentTarget.ownerSVGElement - if (!svg || seriesPoints.length === 0) { - return - } - - const bounds = svg.getBoundingClientRect() - const pointerX = ((event.clientX - bounds.left) / bounds.width) * width - const closestPoint = seriesPoints.reduce((closest, point) => { - return Math.abs(getX(point.date) - pointerX) < Math.abs(getX(closest.date) - pointerX) ? point : closest - }) - setActiveTrend({pointId: closestPoint.id, treatment}) - } - - return ( -
- - {metric.label} - - - {[max, (max + min) / 2, min].map(value => { - const y = getY(value) - return ( - - - - {numberFormatter.format(value)} - - - ) - })} - {dates.map(date => ( - - {date.slice(5)} - - ))} - - Run date - - {series.map((seriesName, index) => { - const seriesPoints = points - .filter(point => getSeriesName(point) === seriesName) - .toSorted((a, b) => a.date.localeCompare(b.date)) - const controlPoints = seriesPoints.filter(point => point.metrics[metric.id].controlValue !== null) - const recommendedPoints = seriesPoints.filter(point => point.metrics[metric.id].value !== null) - const style = lineStyles[index % lineStyles.length] - const recommendedPath = recommendedPoints - .map((point, pointIndex) => { - return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(point.metrics[metric.id].value ?? 0)}` - }) - .join(' ') - const controlPath = controlPoints - .map((point, pointIndex) => { - return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY( - point.metrics[metric.id].controlValue ?? 0, - )}` - }) - .join(' ') - - return ( - - {controlPoints.length > 0 ? ( - <> - - setActiveTrend(null)} - onFocus={() => { - const pointId = controlPoints.at(-1)?.id - setActiveTrend(pointId ? {pointId, treatment: 'control'} : null) - }} - onMouseLeave={() => setActiveTrend(null)} - onMouseMove={event => activateClosestPoint(event, controlPoints, 'control')} - stroke="transparent" - strokeWidth="16" - tabIndex={0} - /> - - ) : null} - {recommendedPoints.length > 0 ? ( - <> - - setActiveTrend(null)} - onFocus={() => { - const pointId = recommendedPoints.at(-1)?.id - setActiveTrend(pointId ? {pointId, treatment: 'recommended'} : null) - }} - onMouseLeave={() => setActiveTrend(null)} - onMouseMove={event => activateClosestPoint(event, recommendedPoints, 'recommended')} - stroke="transparent" - strokeWidth="16" - tabIndex={0} - /> - - ) : null} - {seriesPoints.map(point => { - const value = point.metrics[metric.id] - return ( - - ) - })} - - ) - })} - {activePoint && activeMetric && activeRaw ? ( - - - - {getSeriesName(activePoint)} · {activeTrend?.treatment === 'control' ? 'Control' : 'Recommended'} - - - {activePoint.date}: {activeRaw} - {activeTrend?.treatment === 'recommended' ? ` (${formatChange(activeMetric.change)})` : ''} - - - ) : null} - -
- ) -} - -function ControlTable({ - dates, - metric, - points, - series, -}: { - dates: Array - metric: (typeof metrics)[number] - points: Array - series: Array -}) { - const rows = series.map(seriesName => ({ - id: seriesName, - model: seriesName, - values: Object.fromEntries( - dates.map(date => { - const point = points.find(item => getSeriesName(item) === seriesName && item.date === date) - return [date, point?.metrics[metric.id]] - }), - ), - })) - - return ( - ({ - id: date, - header: () => , - field: 'values' as const, - align: 'end' as const, - renderCell: (row: (typeof rows)[number]) => { - const value = row.values[date] - - return value ? ( - - {value.raw} ({formatChange(value.change)}) - - ) : ( - '—' - ) - }, - })), - ]} - data={rows} - /> - ) -} - -export function BaselineTrends({points, scenarioId}: {points: Array; scenarioId: string}) { - const [heatmapMetricId, setHeatmapMetricId] = useState('tests') - const [selectedSeries, setSelectedSeries] = useState(null) - const [hoveredSeries, setHoveredSeries] = useState(null) - const [selectedTreatment, setSelectedTreatment] = useState<'recommended' | 'control' | null>(null) - const [hoveredTreatment, setHoveredTreatment] = useState<'recommended' | 'control' | null>(null) - const scenarioPoints = points.filter(point => point.scenarioId === scenarioId) - const dates = [...new Set(scenarioPoints.map(point => point.date))].toSorted() - const series = [...new Set(scenarioPoints.map(getSeriesName))].toSorted() - const heatmapMetric = metrics.find(metric => metric.id === heatmapMetricId) ?? metrics[0] - const highlightedSeries = hoveredSeries ?? selectedSeries - const highlightedTreatment = hoveredTreatment ?? selectedTreatment - const rawTrendRows = scenarioPoints - .toSorted((a, b) => a.date.localeCompare(b.date) || getSeriesName(a).localeCompare(getSeriesName(b))) - .map(point => ({ - ...point, - seriesName: getSeriesName(point), - })) - - if (scenarioPoints.length === 0) { - return null - } - - return ( -
- - - - Trends - - - Strong lines show Recommended results and muted lines show Control over time. The comparison table shows - each Recommended result relative to Control. - - - -
-
    - {series.map((seriesName, index) => { - const style = lineStyles[index % lineStyles.length] - return ( -
  • - -
    • - ) - })} -
-
    -
  • - -
    • -
  • - -
    • -
-
-
- {metrics.map(metric => ( - - ))} -
- - - - - Change from Control - - - Recommended results are shown first, followed by the percent change from Control in parentheses. - - - - Metric - - - - - -
- - View raw trend data - -
- , - }, - { - id: 'model', - header: 'Model', - field: 'seriesName', - rowHeader: true, - }, - ...metrics.map(metric => ({ - id: metric.id, - header: metric.label, - field: 'metrics' as const, - align: 'end' as const, - renderCell: (row: (typeof rawTrendRows)[number]) => { - const value = row.metrics[metric.id] - return ( - - {value.raw} ({formatChange(value.change)}) - - ) - }, - })), - ]} - data={rawTrendRows} - /> -
-
- - -
- ) -} diff --git a/website/src/app/components/BenchmarkOverview.module.css b/website/src/app/components/BenchmarkOverview.module.css new file mode 100644 index 00000000..95d258e3 --- /dev/null +++ b/website/src/app/components/BenchmarkOverview.module.css @@ -0,0 +1,8 @@ +.metric { + font: var(--text-codeInline-shorthand); + white-space: nowrap; +} + +.change { + color: var(--fgColor-muted); +} diff --git a/website/src/app/components/BenchmarkOverview.tsx b/website/src/app/components/BenchmarkOverview.tsx new file mode 100644 index 00000000..fab67598 --- /dev/null +++ b/website/src/app/components/BenchmarkOverview.tsx @@ -0,0 +1,142 @@ +'use client' + +import {Stack} from '@primer/react' +import {Blankslate, DataTable, Table} from '@primer/react/experimental' +import type {Route} from 'next' +import type {BenchmarkOverviewData} from '../../benchmark-results' +import type {Benchmark} from '../../benchmarks' +import {Link} from '../../components/Link' +import {BenchmarkTrends} from './BenchmarkTrends' +import styles from './BenchmarkOverview.module.css' + +type BenchmarkOverviewResult = BenchmarkOverviewData['results'][number] + +function Metric({value}: {value: string}) { + const match = /^(.*) \((.*)\)$/.exec(value) + if (!match) { + return value + } + + return ( + + {match[1]} ({match[2]}) + + ) +} + +function BenchmarkResultsTable({ + benchmark, + date, + results, +}: { + benchmark: Benchmark + date: string | null + results: Array +}) { + if (results.length === 0) { + return ( + <> +

{benchmark.name} benchmark

+ + No results + No results have been recorded for this benchmark yet. + + + ) + } + + return ( + + + {benchmark.name} benchmark + + + Each metric is the Benchmark result across all capabilities, followed by the percent change from Control in + parentheses. Models are ranked by test performance, followed by resource usage. + {date ? ( + <> + {' '} + Latest results:{' '} + + + + . + + ) : null} + + { + return + }, + }, + { + id: 'output-tokens', + header: 'Output tokens', + field: 'comparison', + align: 'end', + renderCell: row => { + return + }, + }, + { + id: 'premium-requests', + header: 'Premium requests', + field: 'comparison', + align: 'end', + renderCell: row => { + return + }, + }, + { + id: 'session-time', + header: 'Session time', + field: 'comparison', + align: 'end', + renderCell: row => { + return + }, + }, + { + id: 'api-time', + header: 'API time', + field: 'comparison', + align: 'end', + renderCell: row => { + return + }, + }, + ]} + data={results} + /> + + ) +} + +export function BenchmarkOverview({benchmark, overview}: {benchmark: Benchmark; overview: BenchmarkOverviewData}) { + return ( + + + + + ) +} diff --git a/website/src/app/components/BaselineTrends.module.css b/website/src/app/components/BenchmarkTrends.module.css similarity index 88% rename from website/src/app/components/BaselineTrends.module.css rename to website/src/app/components/BenchmarkTrends.module.css index fbe85381..993cfdb8 100644 --- a/website/src/app/components/BaselineTrends.module.css +++ b/website/src/app/components/BenchmarkTrends.module.css @@ -3,10 +3,7 @@ } .change { - display: inline-block; - min-width: 8ch; color: var(--fgColor-muted); - text-align: right; } .chartGrid { @@ -58,6 +55,11 @@ stroke: var(--focus-outline-color); } +.pointHitTarget { + cursor: crosshair; + fill: transparent; +} + .tooltip { pointer-events: none; } @@ -89,6 +91,21 @@ list-style: none; } +.filters { + display: flex; + flex-wrap: wrap; + align-items: start; + gap: var(--stack-gap-normal); +} + +.trendsHeader { + display: flex; + flex-wrap: wrap; + align-items: start; + justify-content: space-between; + gap: var(--stack-gap-normal); +} + .legendGroups { display: flex; flex-wrap: wrap; @@ -122,17 +139,11 @@ } } -.legendItem { - display: flex; - align-items: center; - gap: var(--stack-gap-condensed); - font: var(--text-body-shorthand-small); -} - .metricValue { display: inline-flex; width: 100%; justify-content: flex-end; + gap: 0.5ch; font: var(--text-codeInline-shorthand); white-space: nowrap; } diff --git a/website/src/app/components/BenchmarkTrends.tsx b/website/src/app/components/BenchmarkTrends.tsx new file mode 100644 index 00000000..a1fa0d41 --- /dev/null +++ b/website/src/app/components/BenchmarkTrends.tsx @@ -0,0 +1,776 @@ +'use client' + +import {Button, FormControl, Heading, Select, Stack, Text} from '@primer/react' +import {DataTable, Table} from '@primer/react/experimental' +import {type MouseEvent, useState} from 'react' +import type {BenchmarkTrendMetricId, BenchmarkTrendPoint} from '../../benchmark-results' +import type {Benchmark} from '../../benchmarks' +import styles from './BenchmarkTrends.module.css' + +const metrics: Array<{ + id: BenchmarkTrendMetricId + label: string +}> = [ + {id: 'tests', label: 'Tests passed'}, + {id: 'outputTokens', label: 'Output tokens'}, + {id: 'premiumRequests', label: 'Premium requests'}, + {id: 'sessionTime', label: 'Session time'}, + {id: 'apiTime', label: 'API time'}, +] + +const lineStyles = [ + {color: 'var(--data-blue-color-emphasis, var(--data-blue-color))', dash: undefined}, + {color: 'var(--data-green-color-emphasis, var(--data-green-color))', dash: '7 3'}, + {color: 'var(--data-orange-color-emphasis, var(--data-orange-color))', dash: '2 3'}, + {color: 'var(--data-pink-color-emphasis, var(--data-pink-color))', dash: '10 3 2 3'}, + {color: 'var(--data-yellow-color-emphasis, var(--data-yellow-color))', dash: '4 4'}, + {color: 'var(--data-red-color-emphasis, var(--data-red-color))', dash: '12 4'}, +] + +const numberFormatter = new Intl.NumberFormat('en-US', { + maximumFractionDigits: 1, +}) + +type TrendTreatment = 'benchmark' | 'control' + +function getSeriesName(point: BenchmarkTrendPoint): string { + return point.reasoningEffort ? `${point.model} (${point.reasoningEffort})` : point.model +} + +function formatChange(change: number | null): string { + if (change === null) { + return 'N/A' + } + + return `${change > 0 ? '+' : ''}${numberFormatter.format(change)}%` +} + +function formatAxisValue(metricId: BenchmarkTrendMetricId, value: number): string { + if (metricId === 'tests') { + return `${Math.round(value)}%` + } + + if (metricId === 'sessionTime' || metricId === 'apiTime') { + return `${numberFormatter.format(value)}s` + } + + return numberFormatter.format(value) +} + +function getAxisDomain( + metricId: BenchmarkTrendMetricId, + rawMin: number, + rawMax: number, +): { + min: number + max: number +} { + const range = rawMax - rawMin + const margin = (range || Math.max(rawMax, 1)) * 0.1 + + if (metricId !== 'tests') { + return { + min: Math.max(0, rawMin - margin), + max: rawMax + margin, + } + } + + return { + min: 0, + max: 100, + } +} + +function TrendChart({ + dates, + metric, + points, + series, + highlightedSeries, + highlightedTreatment, +}: { + dates: Array + metric: (typeof metrics)[number] + points: Array + series: Array + highlightedSeries: string | null + highlightedTreatment: TrendTreatment | null +}) { + const [activeTrend, setActiveTrend] = useState<{ + pointId: string + treatment: TrendTreatment + } | null>(null) + const width = 520 + const height = 240 + const padding = {top: 16, right: 16, bottom: 52, left: 48} + const plotWidth = width - padding.left - padding.right + const plotHeight = height - padding.top - padding.bottom + const values = points.flatMap(point => { + const value = point.metrics[metric.id] + return [value.value, value.controlValue].filter((item): item is number => { + return item !== null + }) + }) + const rawMin = values.length > 0 ? Math.min(...values) : 0 + const rawMax = values.length > 0 ? Math.max(...values) : 0 + const {min, max} = getAxisDomain(metric.id, rawMin, rawMax) + const axisValues = metric.id === 'tests' ? [max, Math.round((max + min) / 2), min] : [max, (max + min) / 2, min] + const getX = (date: string): number => { + return padding.left + (dates.length === 1 ? plotWidth / 2 : (dates.indexOf(date) / (dates.length - 1)) * plotWidth) + } + const getY = (value: number): number => { + return padding.top + (1 - (value - min) / (max - min)) * plotHeight + } + const activePoint = points.find(point => { + return point.id === activeTrend?.pointId + }) + const activeMetric = activePoint?.metrics[metric.id] + const activeValue = + activeTrend?.treatment === 'control' + ? (activeMetric?.controlValue ?? undefined) + : (activeMetric?.value ?? undefined) + const activeRaw = activeTrend?.treatment === 'control' ? activeMetric?.controlRaw : activeMetric?.raw + const tooltipWidth = 248 + const tooltipHeight = 52 + const tooltipX = activePoint + ? Math.min(Math.max(getX(activePoint.date) - tooltipWidth / 2, padding.left), width - padding.right - tooltipWidth) + : 0 + const tooltipY = + activePoint && activeValue !== undefined + ? Math.max(padding.top, Math.min(getY(activeValue) - tooltipHeight - 10, height - padding.bottom - tooltipHeight)) + : 0 + const tooltipId = `${metric.id}-chart-tooltip` + const controlOpacity = highlightedTreatment === 'benchmark' ? 0.15 : highlightedTreatment === 'control' ? 1 : 0.4 + const benchmarkOpacity = highlightedTreatment === 'control' ? 0.15 : 1 + const activateClosestPoint = ( + event: MouseEvent, + seriesPoints: Array, + treatment: TrendTreatment, + ) => { + const svg = event.currentTarget.ownerSVGElement + if (!svg || seriesPoints.length === 0) { + return + } + + const bounds = svg.getBoundingClientRect() + const pointerX = ((event.clientX - bounds.left) / bounds.width) * width + const closestPoint = seriesPoints.reduce((closest, point) => { + return Math.abs(getX(point.date) - pointerX) < Math.abs(getX(closest.date) - pointerX) ? point : closest + }) + setActiveTrend({pointId: closestPoint.id, treatment}) + } + + return ( +
+ + {metric.label} + + + {axisValues.map(value => { + const y = getY(value) + return ( + + + + {formatAxisValue(metric.id, value)} + + + ) + })} + {dates.map(date => { + return ( + + {date.slice(5)} + + ) + })} + + Run date + + {series.map((seriesName, index) => { + const seriesPoints = points + .filter(point => { + return getSeriesName(point) === seriesName + }) + .toSorted((a, b) => { + return a.date.localeCompare(b.date) + }) + const controlPoints = seriesPoints.filter(point => { + return point.metrics[metric.id].controlValue !== null + }) + const benchmarkPoints = seriesPoints.filter(point => { + return point.metrics[metric.id].value !== null + }) + const style = lineStyles[index % lineStyles.length] + const controlPath = controlPoints + .map((point, pointIndex) => { + return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY( + point.metrics[metric.id].controlValue ?? 0, + )}` + }) + .join(' ') + const benchmarkPath = benchmarkPoints + .map((point, pointIndex) => { + return `${pointIndex === 0 ? 'M' : 'L'} ${getX(point.date)} ${getY(point.metrics[metric.id].value ?? 0)}` + }) + .join(' ') + + return ( + + {controlPoints.length > 0 ? ( + <> + + { + setActiveTrend(null) + }} + onFocus={() => { + const pointId = controlPoints.at(-1)?.id + setActiveTrend(pointId ? {pointId, treatment: 'control'} : null) + }} + onMouseLeave={() => { + setActiveTrend(null) + }} + onMouseMove={event => { + activateClosestPoint(event, controlPoints, 'control') + }} + stroke="transparent" + strokeWidth="16" + tabIndex={0} + /> + + ) : null} + {benchmarkPoints.length > 0 ? ( + <> + + { + setActiveTrend(null) + }} + onFocus={() => { + const pointId = benchmarkPoints.at(-1)?.id + setActiveTrend(pointId ? {pointId, treatment: 'benchmark'} : null) + }} + onMouseLeave={() => { + setActiveTrend(null) + }} + onMouseMove={event => { + activateClosestPoint(event, benchmarkPoints, 'benchmark') + }} + stroke="transparent" + strokeWidth="16" + tabIndex={0} + /> + + ) : null} + {seriesPoints.map(point => { + const value = point.metrics[metric.id] + return ( + + ) + })} + + ) + })} + {activePoint && activeMetric && activeRaw ? ( + + + + {`${getSeriesName(activePoint)} - ${activeTrend?.treatment === 'control' ? 'Control' : 'Benchmark'}`} + + + {`${activePoint.date}: ${activeRaw}${ + activeTrend?.treatment === 'benchmark' ? ` (${formatChange(activeMetric.change)})` : '' + }`} + + + ) : null} + +
+ ) +} + +export function BenchmarkTrends({ + capabilities, + points, +}: { + capabilities: Benchmark['capabilities'] + points: Array +}) { + const [tableMetricId, setTableMetricId] = useState('tests') + const [selectedCapabilityId, setSelectedCapabilityId] = useState('') + const [selectedScenarioId, setSelectedScenarioId] = useState('') + const [selectedSeries, setSelectedSeries] = useState(null) + const [hoveredSeries, setHoveredSeries] = useState(null) + const [selectedTreatment, setSelectedTreatment] = useState(null) + const [hoveredTreatment, setHoveredTreatment] = useState(null) + const scenarios = [ + ...new Set( + (selectedCapabilityId + ? (capabilities.find(capability => { + return capability.name === selectedCapabilityId + })?.scenarios ?? []) + : capabilities.flatMap(capability => { + return capability.scenarios + }) + ).map(scenario => { + return scenario.id + }), + ), + ].toSorted() + const filteredPoints = points.filter(point => { + return point.capabilityId === (selectedCapabilityId || null) && point.scenarioId === (selectedScenarioId || null) + }) + const dates = [ + ...new Set( + filteredPoints.map(point => { + return point.date + }), + ), + ].toSorted() + const series = [ + ...new Set( + filteredPoints.map(point => { + return getSeriesName(point) + }), + ), + ].toSorted() + const tableMetric = + metrics.find(metric => { + return metric.id === tableMetricId + }) ?? metrics[0] + const highlightedSeries = hoveredSeries ?? selectedSeries + const highlightedTreatment = hoveredTreatment ?? selectedTreatment + const comparisonRows = series.map(seriesName => { + return { + id: seriesName, + model: seriesName, + values: Object.fromEntries( + dates.map(date => { + const point = filteredPoints.find(candidate => { + return getSeriesName(candidate) === seriesName && candidate.date === date + }) + return [date, point?.metrics[tableMetric.id]] + }), + ), + } + }) + const rawRows = filteredPoints + .toSorted((a, b) => { + return a.date.localeCompare(b.date) || getSeriesName(a).localeCompare(getSeriesName(b)) + }) + .map(point => { + return {...point, seriesName: getSeriesName(point)} + }) + + if (points.length === 0) { + return null + } + + return ( +
+ +
+ + + Trends + + + Strong lines show Benchmark results and muted lines show Control over time. + + +
+ + Capability + + + + Scenario + + +
+
+ +
+
    + {series.map((seriesName, index) => { + const style = lineStyles[index % lineStyles.length] + return ( +
  • + +
    • + ) + })} +
+
    +
  • + +
    • +
  • + +
    • +
+
+
+ {metrics.map(metric => { + return ( + + ) + })} +
+ + + + Change from Control + + + Benchmark results are shown first, followed by the percent change from Control in parentheses. + + + + Metric + + + + { + return { + id: date, + header: () => { + return + }, + field: 'values' as const, + align: 'end' as const, + renderCell: (row: (typeof comparisonRows)[number]) => { + const value = row.values[date] + return value ? ( + + {value.raw} ({formatChange(value.change)}) + + ) : ( + 'N/A' + ) + }, + } + }), + ]} + data={comparisonRows} + /> + +
+ + View raw trend data + +
+ { + return + }, + }, + { + id: 'model', + header: 'Model', + field: 'seriesName', + rowHeader: true, + }, + ...metrics.map(metric => { + return { + id: metric.id, + header: metric.label, + field: 'metrics' as const, + align: 'end' as const, + renderCell: (row: (typeof rawRows)[number]) => { + const value = row.metrics[metric.id] + return ( + + {value.raw} ({formatChange(value.change)}) + + ) + }, + } + }), + ]} + data={rawRows} + /> +
+
+ +
+ ) +} diff --git a/website/src/app/components/Index.module.css b/website/src/app/components/Index.module.css deleted file mode 100644 index 7190d400..00000000 --- a/website/src/app/components/Index.module.css +++ /dev/null @@ -1,15 +0,0 @@ -.layout { - min-height: calc(100dvh - var(--header-height) - (2 * var(--stack-padding-normal))); -} - -.metric { - font: var(--text-codeInline-shorthand); - white-space: nowrap; -} - -.change { - display: inline-block; - min-width: 8ch; - color: var(--fgColor-muted); - text-align: right; -} diff --git a/website/src/app/components/Index.tsx b/website/src/app/components/Index.tsx deleted file mode 100644 index 26998085..00000000 --- a/website/src/app/components/Index.tsx +++ /dev/null @@ -1,220 +0,0 @@ -'use client' - -import {NavList, PageLayout, Stack} from '@primer/react' -import {DataTable, Table, Blankslate} from '@primer/react/experimental' -import type {Route} from 'next' -import NextLink from 'next/link' -import {Link} from '../../components/Link' -import {BaselineTrends, type BaselineTrendPoint} from './BaselineTrends' -import styles from './Index.module.css' - -type BaselineMetric = { - raw: string - change: string | null -} - -export type BaselineResult = { - id: string - model: string - reasoningEffort: string - tests: BaselineMetric - turns: BaselineMetric - outputTokens: BaselineMetric - premiumRequests: BaselineMetric - apiDuration: BaselineMetric - sessionDuration: BaselineMetric - toolCalls: BaselineMetric -} - -export type BaselineComparison = { - id: string - scenarioId: string - results: Array -} - -function Metric({value}: {value: BaselineMetric}) { - if (value.raw === '—') { - return '—' - } - - return ( - - {value.raw} ({value.change ?? '—'}) - - ) -} - -export function BaselineResultsTable({ - description, - heading, - headingId, - headingLevel = 'h2', - results, -}: { - description?: React.ReactNode - heading: React.ReactNode - headingId: string - headingLevel?: 'h1' | 'h2' - results: Array -}) { - const descriptionId = `${headingId}-description` - - return ( - - - {heading} - - {description ? ( - - {description} - - ) : null} - , - }, - { - id: 'turns', - header: 'Turns', - field: 'turns', - align: 'end', - width: 'auto', - renderCell: row => , - }, - { - id: 'output-tokens', - header: 'Output tokens', - field: 'outputTokens', - align: 'end', - width: 'auto', - renderCell: row => , - }, - { - id: 'premium-requests', - header: 'Premium requests', - field: 'premiumRequests', - align: 'end', - width: 'auto', - renderCell: row => , - }, - { - id: 'api-time', - header: 'API time', - field: 'apiDuration', - align: 'end', - width: 'auto', - renderCell: row => , - }, - { - id: 'session-time', - header: 'Session time', - field: 'sessionDuration', - align: 'end', - width: 'auto', - renderCell: row => , - }, - { - id: 'tool-calls', - header: 'Tool calls', - field: 'toolCalls', - align: 'end', - width: 'auto', - renderCell: row => , - }, - ]} - data={results} - /> - - ) -} - -export function Index({ - baseline, - baselineTrends, - selectedScenarioId, -}: { - baseline: Array | null - baselineTrends: Array - selectedScenarioId?: string -}) { - const selectedScenario = baseline?.find(scenario => scenario.scenarioId === selectedScenarioId) ?? baseline?.[0] - - return ( - -
- {baseline ? ( - - - - {baseline.map(scenario => ( - - {scenario.scenarioId} - - ))} - - - - {selectedScenario ? ( - - - Recommended results are shown first, followed by the percent change from Control in parentheses. - Positive means the raw value increased and negative means it decreased. Models are ranked by - test pass rate within each scenario. - - } - heading={ - {selectedScenario.scenarioId} - } - headingId={`baseline-${selectedScenario.scenarioId}-heading`} - headingLevel="h1" - results={selectedScenario.results} - /> - - - ) : null} - - - ) : ( - <> -

Baseline

- - No results - - No baseline results have been recorded yet. Run the baseline tests to see results here. - - - - )} -
- - ) -} diff --git a/website/src/app/components/PageHeader.tsx b/website/src/app/components/PageHeader.tsx index 2de09f9a..cb210dbd 100644 --- a/website/src/app/components/PageHeader.tsx +++ b/website/src/app/components/PageHeader.tsx @@ -25,10 +25,10 @@ export function PageHeader() { - Baseline + Benchmarks +}) { + return withStandaloneLayout( +
+ + + Benchmarks + + {showViewAll ? ( + + View all + + ) : null} + { + return {row.name} + }, + }, + {id: 'description', header: 'Description', field: 'description', maxWidth: '60ch'}, + {id: 'models', header: 'Models', field: 'models', align: 'end'}, + {id: 'capabilities', header: 'Capabilities', field: 'capabilities', align: 'end'}, + {id: 'scenarios', header: 'Scenarios', field: 'scenarios', align: 'end'}, + ]} + data={benchmarks.map(benchmark => ({ + id: benchmark.id, + name: benchmark.name, + description: benchmark.description, + models: benchmark.models.length, + capabilities: benchmark.capabilities.length, + scenarios: new Set( + benchmark.capabilities.flatMap(capability => { + return capability.scenarios.map(scenario => { + return scenario.id + }) + }), + ).size, + }))} + /> + +
, + standalone, + ) +} diff --git a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx b/website/src/app/components/RunDetailsPage.tsx similarity index 92% rename from website/src/app/experiments/[id]/runs/[date]/components/Page.tsx rename to website/src/app/components/RunDetailsPage.tsx index b20d3218..28ee1d22 100644 --- a/website/src/app/experiments/[id]/runs/[date]/components/Page.tsx +++ b/website/src/app/components/RunDetailsPage.tsx @@ -2,46 +2,13 @@ import {CheckCircleFillIcon, CopilotIcon, PersonIcon, XCircleFillIcon} from '@primer/octicons-react' import {Breadcrumbs, FormControl, Select, Stack, UnderlineNav} from '@primer/react' -import type {Experiment} from '../../../../../../experiments' +import type {RunDetails, TranscriptEntry, WalkthroughDataUrl} from '../../run-details' import type {Route} from 'next' import Link from 'next/link' import Image from 'next/image' import {useState} from 'react' -import type {WalkthroughDataUrl} from '../page' -type TranscriptEntry = { - id: string - label: string - timestamp?: string - content: string -} - -type RunResult = { - id: string - scenarioId: string - treatment: string - model: string - reasoningEffort?: string - testsPassed: number - totalTests: number - turns: number - outputTokens: number - premiumRequests: number - totalApiDurationMs: number - sessionDurationMs: number - tests: Array<{ - fullName: string - status: string - description?: string - }> - transcript: Array - walkthrough: WalkthroughDataUrl -} - -type RunDetails = { - date: string - results: Array -} +type RunResult = RunDetails['results'][number] type ScenarioResultGroup = { scenarioId: string @@ -394,26 +361,32 @@ function ScenarioResults({group, index}: {group: ScenarioResultGroup; index: num } type Props = { - experiment: Experiment + resource: { + id: string + name: string + collectionLabel: string + collectionHref: Route + href: Route + } run: RunDetails } -export function Page({experiment, run}: Props) { +export function RunDetailsPage({resource, run}: Props) { const resultGroups = groupResultsByScenario(run.results) return (
- - Experiments + + {resource.collectionLabel} - - {experiment.id} + + {resource.id} {run.date} -

Run results for {experiment.name}

+

Run results for {resource.name}

{resultGroups.map((group, index) => { return @@ -423,5 +396,3 @@ export function Page({experiment, run}: Props) { ) } - -export type {RunDetails, TranscriptEntry} diff --git a/website/src/app/experiments/[id]/runs/[date]/page.tsx b/website/src/app/experiments/[id]/runs/[date]/page.tsx index 0b3244d0..02f524cd 100644 --- a/website/src/app/experiments/[id]/runs/[date]/page.tsx +++ b/website/src/app/experiments/[id]/runs/[date]/page.tsx @@ -1,16 +1,11 @@ -import type {AgentEvalOutput} from '@primer/agent-eval/output' +import type {Route} from 'next' +import {notFound} from 'next/navigation' import {get as getExperiment} from '../../../../../experiments' +import {createExperimentRunDetails} from '../../../../../run-details' import {get as getRun, list as listRuns} from '../../../../../runs' -import {notFound} from 'next/navigation' -import fs from 'node:fs/promises' -import path from 'node:path' -import {Page} from './components/Page' -import type {RunDetails, TranscriptEntry} from './components/Page' -import type {Walkthrough} from '../../../../../../../packages/agent-eval/src/treatment' +import {RunDetailsPage} from '../../../../components/RunDetailsPage' const EMPTY_RUN_PARAM = '__no-runs__' -const REPOSITORY_ROOT = path.resolve(process.cwd(), '..') -const ARTIFACTS_DIRECTORY = path.join(REPOSITORY_ROOT, 'artifacts') type RunPageProps = { params: Promise<{ @@ -19,235 +14,6 @@ type RunPageProps = { }> } -type LogMessage = AgentEvalOutput['results'][number]['assistant']['logs'][number] - -function asRecord(value: unknown): Record | null { - return typeof value === 'object' && value !== null ? (value as Record) : null -} - -function getString(record: Record | null, key: string): string | undefined { - const value = record?.[key] - return typeof value === 'string' ? value : undefined -} - -function createTranscript(logs: Array): Array { - const entries: Array = [] - const messageEntries = new Map() - const reasoningEntries = new Map() - const toolNames = new Map() - - for (const [index, message] of logs.entries()) { - const record = asRecord(message) - const data = asRecord(record?.data) - const timestamp = getString(record, 'timestamp') - const id = getString(record, 'id') ?? `${message.type}-${index}` - - switch (message.type) { - case 'user.message': { - const content = getString(data, 'content') - if (content) { - entries.push({id, label: 'User', timestamp, content}) - } - break - } - case 'assistant.message_delta': { - const messageId = getString(data, 'messageId') - const delta = getString(data, 'deltaContent') - if (!messageId || !delta) { - break - } - - let entry = messageEntries.get(messageId) - if (!entry) { - entry = {id, label: 'Assistant', timestamp, content: ''} - messageEntries.set(messageId, entry) - entries.push(entry) - } - entry.content += delta - break - } - case 'assistant.message': { - const messageId = getString(data, 'messageId') - const content = getString(data, 'content') - const entry = messageId ? messageEntries.get(messageId) : undefined - if (entry) { - if (content) { - entry.content = content - } - } else if (content) { - entries.push({id, label: 'Assistant', timestamp, content}) - } - break - } - case 'assistant.reasoning_delta': { - const reasoningId = getString(data, 'reasoningId') - const delta = getString(data, 'deltaContent') - if (!reasoningId || !delta) { - break - } - - let entry = reasoningEntries.get(reasoningId) - if (!entry) { - entry = {id, label: 'Reasoning', timestamp, content: ''} - reasoningEntries.set(reasoningId, entry) - entries.push(entry) - } - entry.content += delta - break - } - case 'assistant.reasoning': { - const reasoningId = getString(data, 'reasoningId') - const content = getString(data, 'content') - const entry = reasoningId ? reasoningEntries.get(reasoningId) : undefined - if (entry) { - if (content) { - entry.content = content - } - } else if (content) { - entries.push({id, label: 'Reasoning', timestamp, content}) - } - break - } - case 'tool.execution_start': { - const toolName = getString(data, 'toolName') ?? 'Unknown tool' - const toolCallId = getString(data, 'toolCallId') - if (toolCallId) { - toolNames.set(toolCallId, toolName) - } - entries.push({id, label: `Tool call: ${toolName}`, timestamp, content: 'Started'}) - break - } - case 'tool.execution_complete': { - const toolCallId = getString(data, 'toolCallId') - const toolName = toolCallId ? toolNames.get(toolCallId) : undefined - entries.push({ - id, - label: `Tool result: ${toolName ?? 'Unknown tool'}`, - timestamp, - content: data?.success === true ? 'Completed successfully' : 'Failed', - }) - break - } - case 'session.info': { - const content = getString(data, 'message') - if (content) { - entries.push({id, label: 'Session', timestamp, content}) - } - break - } - case 'session.task_complete': { - const content = getString(data, 'summary') - if (content) { - entries.push({id, label: 'Summary', timestamp, content}) - } - break - } - } - } - - return entries.filter(entry => entry.content.length > 0) -} - -async function getArtifactDataUrl(artifactPath: string | undefined, mimeType: string): Promise { - if (!artifactPath) { - return undefined - } - - const absolutePath = path.isAbsolute(artifactPath) ? artifactPath : path.resolve(REPOSITORY_ROOT, artifactPath) - const relativePath = path.relative(ARTIFACTS_DIRECTORY, absolutePath) - if (relativePath.startsWith('..') || path.isAbsolute(relativePath)) { - return undefined - } - - try { - const contents = await fs.readFile(absolutePath) - return `data:${mimeType};base64,${contents.toString('base64')}` - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return undefined - } - throw error - } -} - -function getImageMimeType(artifactPath: string): string { - const extension = path.extname(artifactPath).toLowerCase() - if (extension === '.jpg' || extension === '.jpeg') { - return 'image/jpeg' - } - return 'image/png' -} - -export type WalkthroughDataUrl = - | { - type: 'Unavailable' - } - | {type: 'Screenshot'; screenshot: string} - | {type: 'Screenshots'; screenshots: Array} - | {type: 'Video'; video: string} - -async function getWalkthroughDataUrls(walkthrough: Walkthrough): Promise { - if (walkthrough.type === 'Screenshot') { - return { - type: 'Screenshot', - screenshot: await getArtifactDataUrl(walkthrough.filepath, getImageMimeType(walkthrough.filepath)), - } - } - - if (walkthrough.type === 'Screenshots') { - const sources = await Promise.all( - walkthrough.screenshots.map(artifactPath => getArtifactDataUrl(artifactPath, getImageMimeType(artifactPath))), - ) - return { - type: 'Screenshots', - screenshots: sources.filter((source): source is string => source !== undefined), - } - } - - if (walkthrough.type === 'Video') { - return { - type: 'Video', - video: await getArtifactDataUrl(walkthrough.filepath, 'video/webm'), - } - } - - return { - type: 'Unavailable', - } -} - -async function createRunDetails(date: string, output: AgentEvalOutput): Promise { - const treatments = new Map(output.treatments.map(treatment => [treatment.id, treatment.config.name])) - return { - date, - results: await Promise.all( - output.results.map(async result => ({ - id: result.id, - scenarioId: result.scenarioId, - treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment', - model: result.model, - reasoningEffort: result.reasoningEffort, - testsPassed: result.testResults.numPassedTests, - totalTests: result.testResults.numTotalTests, - turns: result.assistant.turns, - outputTokens: result.assistant.outputTokens, - premiumRequests: result.assistant.premiumRequests, - totalApiDurationMs: result.assistant.totalApiDurationMs, - sessionDurationMs: result.assistant.sessionDurationMs, - tests: result.testResults.tests.map(test => { - return { - fullName: test.fullName, - status: test.status, - description: test.description, - } - }), - walkthrough: await getWalkthroughDataUrls(result.walkthrough), - transcript: createTranscript(result.assistant.logs), - })), - ), - } -} - export const dynamicParams = false export default async function RunPage(props: RunPageProps) { @@ -256,12 +22,20 @@ export default async function RunPage(props: RunPageProps) { notFound() } - const [experiment, run] = await Promise.all([getExperiment(id), getRun(date)]) - if (run.output.experiment.id !== id) { - notFound() - } + const [experiment, run] = await Promise.all([getExperiment(id), getRun(id, date)]) - return + return ( + + ) } export async function generateStaticParams() { @@ -270,8 +44,10 @@ export async function generateStaticParams() { return [{id: EMPTY_RUN_PARAM, date: EMPTY_RUN_PARAM}] } - return runs.map(run => ({ - id: run.output.experiment.id, - date: run.name, - })) + return runs.map(run => { + return { + id: run.output.experiment.id, + date: run.name, + } + }) } diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx index c48dc822..8f449a16 100644 --- a/website/src/app/layout.tsx +++ b/website/src/app/layout.tsx @@ -7,7 +7,7 @@ export const metadata = { default: 'primer / agent-eval', template: '%s · primer / agent-eval', }, - description: 'View results of experiments with agents across different scenarios with Primer', + description: 'View design system benchmark and experiment results for coding agents', } export default function Layout({children}: {children: React.ReactNode}) { diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx index 3c812b3a..2473472a 100644 --- a/website/src/app/page.tsx +++ b/website/src/app/page.tsx @@ -1,7 +1,7 @@ -import {getBaselinePageData} from '../baseline-results' -import {BaselineOverview} from './components/BaselineOverview' +import {getBenchmarkPageData} from '../benchmark-page-data' +import {BenchmarkOverview} from './components/BenchmarkOverview' export default async function IndexPage() { - const {aggregateResults, aggregateTrends} = await getBaselinePageData() - return + const {benchmark, overview} = await getBenchmarkPageData('design-system') + return } diff --git a/website/src/baseline-results.ts b/website/src/baseline-results.ts deleted file mode 100644 index fcb5e3a1..00000000 --- a/website/src/baseline-results.ts +++ /dev/null @@ -1,477 +0,0 @@ -import type {AgentEvalOutput, AgentEvalOutputResult} from '@primer/agent-eval/output' -import type {BaselineComparison, BaselineResult} from './app/components/Index' -import type {BaselineTrendPoint} from './app/components/BaselineTrends' -import {list as listRuns} from './runs' - -type TreatmentResults = { - control?: AgentEvalOutputResult - baseline?: AgentEvalOutputResult -} - -type ModelTreatmentResults = TreatmentResults & { - id: string - model: string - reasoningEffort: string -} - -type MetricValue = { - raw: string - change: string | null -} - -const percentFormatter = new Intl.NumberFormat('en-US', { - maximumFractionDigits: 1, -}) - -const numberFormatter = new Intl.NumberFormat('en-US') - -function getResultKey(scenarioId: string, model: string, reasoningEffort: string | undefined) { - return JSON.stringify([scenarioId, model, reasoningEffort ?? '']) -} - -function getModelResultKey(model: string, reasoningEffort: string | undefined) { - return JSON.stringify([model, reasoningEffort ?? '']) -} - -function getPercentChange(control: number | undefined, baseline: number | undefined): string | null { - const change = getPercentChangeValue(control, baseline) - if (change === null) { - return null - } - - return `${change > 0 ? '+' : ''}${percentFormatter.format(change)}%` -} - -function getPercentChangeValue(control: number | undefined, baseline: number | undefined): number | null { - if (control === undefined || baseline === undefined || (control === 0 && baseline !== 0)) { - return null - } - - if (control === baseline) { - return 0 - } - - return ((baseline - control) / Math.abs(control)) * 100 -} - -function getTestPassRate(result: AgentEvalOutputResult | undefined) { - if (!result) { - return undefined - } - - if (result.testResults.numTotalTests === 0) { - return 0 - } - - return result.testResults.numPassedTests / result.testResults.numTotalTests -} - -function countToolCalls(result: AgentEvalOutputResult | undefined) { - if (!result) { - return undefined - } - - return Object.values(result.assistant.tools).reduce((total, count) => total + count, 0) -} - -function average(values: Array) { - const recorded = values.filter(value => value !== undefined) - if (recorded.length === 0) { - return undefined - } - - return recorded.reduce((total, value) => total + value, 0) / recorded.length -} - -function formatMetric( - control: number | undefined, - baseline: number | undefined, - format: (value: number) => string, -): MetricValue { - return { - raw: baseline === undefined ? '—' : format(baseline), - change: getPercentChange(control, baseline), - } -} - -function formatDuration(milliseconds: number): string { - if (milliseconds < 1000) { - return `${numberFormatter.format(milliseconds)}ms` - } - - return `${percentFormatter.format(milliseconds / 1000)}s` -} - -function getBaselineComparisons(output: AgentEvalOutput): Array { - const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control') - const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended') - - if (!controlTreatment || !baselineTreatment) { - throw new Error('The latest baseline run must include Control and Recommended treatments') - } - - const resultsByScenario = new Map>() - for (const scenario of output.scenarios) { - const scenarioResults = new Map() - for (const model of output.experiment.models) { - const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined] - for (const reasoningEffort of reasoningEfforts) { - scenarioResults.set(getModelResultKey(model.name, reasoningEffort), { - id: getResultKey(scenario.id, model.name, reasoningEffort), - model: model.name, - reasoningEffort: reasoningEffort ?? '—', - }) - } - } - resultsByScenario.set(scenario.id, scenarioResults) - } - - for (const result of output.results) { - const treatmentResults = resultsByScenario - .get(result.scenarioId) - ?.get(getModelResultKey(result.model, result.reasoningEffort)) - if (!treatmentResults) { - throw new Error(`Result "${result.id}" does not match a configured scenario, model, and reasoning effort`) - } - - if (result.treatmentId === controlTreatment.id) { - treatmentResults.control = result - } else if (result.treatmentId === baselineTreatment.id) { - treatmentResults.baseline = result - } - } - - return output.scenarios.map(scenario => { - const comparisons = Array.from(resultsByScenario.get(scenario.id)?.values() ?? [], treatmentResults => { - const {id, model, reasoningEffort, control, baseline} = treatmentResults - const baselinePassRate = getTestPassRate(baseline) - - return { - id, - model, - reasoningEffort, - passRate: baselinePassRate, - turnsValue: baseline?.assistant.turns, - tests: { - raw: baseline ? `${baseline.testResults.numPassedTests}/${baseline.testResults.numTotalTests}` : '—', - change: getPercentChange(control?.testResults.numPassedTests, baseline?.testResults.numPassedTests), - }, - turns: formatMetric(control?.assistant.turns, baseline?.assistant.turns, value => - numberFormatter.format(value), - ), - outputTokens: formatMetric(control?.assistant.outputTokens, baseline?.assistant.outputTokens, value => - numberFormatter.format(value), - ), - premiumRequests: formatMetric(control?.assistant.premiumRequests, baseline?.assistant.premiumRequests, value => - numberFormatter.format(value), - ), - apiDuration: formatMetric( - control?.assistant.totalApiDurationMs, - baseline?.assistant.totalApiDurationMs, - formatDuration, - ), - sessionDuration: formatMetric( - control?.assistant.sessionDurationMs, - baseline?.assistant.sessionDurationMs, - formatDuration, - ), - toolCalls: formatMetric(countToolCalls(control), countToolCalls(baseline), value => - numberFormatter.format(value), - ), - } - }) - - return { - id: scenario.id, - scenarioId: scenario.id, - results: comparisons - .toSorted((a, b) => { - if (a.passRate !== b.passRate) { - return (b.passRate ?? -1) - (a.passRate ?? -1) - } - - if (a.turnsValue !== b.turnsValue) { - return (a.turnsValue ?? Number.POSITIVE_INFINITY) - (b.turnsValue ?? Number.POSITIVE_INFINITY) - } - - return a.model.localeCompare(b.model) - }) - .map(comparison => ({ - id: comparison.id, - model: comparison.model, - reasoningEffort: comparison.reasoningEffort, - tests: comparison.tests, - turns: comparison.turns, - outputTokens: comparison.outputTokens, - premiumRequests: comparison.premiumRequests, - apiDuration: comparison.apiDuration, - sessionDuration: comparison.sessionDuration, - toolCalls: comparison.toolCalls, - })), - } - }) -} - -function getAggregateBaselineResults(output: AgentEvalOutput): Array { - const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control') - const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended') - - if (!controlTreatment || !baselineTreatment) { - throw new Error('The latest baseline run must include Control and Recommended treatments') - } - - const results = output.experiment.models.flatMap(model => { - const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined] - return reasoningEfforts.map(reasoningEffort => { - const matches = (result: AgentEvalOutputResult) => - result.model === model.name && result.reasoningEffort === reasoningEffort - const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result)) - const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result)) - const baselinePassRate = average(baselines.map(getTestPassRate)) - const baselinePassedTests = average(baselines.map(result => result.testResults.numPassedTests)) - const baselineTotalTests = average(baselines.map(result => result.testResults.numTotalTests)) - const controlTurns = average(controls.map(result => result.assistant.turns)) - const baselineTurns = average(baselines.map(result => result.assistant.turns)) - - return { - id: getResultKey('aggregate', model.name, reasoningEffort), - model: model.name, - reasoningEffort: reasoningEffort ?? '—', - passRate: baselinePassRate, - turnsValue: baselineTurns, - tests: { - raw: - baselinePassedTests === undefined || baselineTotalTests === undefined - ? '—' - : `${numberFormatter.format(baselinePassedTests)}/${numberFormatter.format(baselineTotalTests)}`, - change: getPercentChange( - average(controls.map(result => result.testResults.numPassedTests)), - baselinePassedTests, - ), - }, - turns: formatMetric(controlTurns, baselineTurns, value => numberFormatter.format(value)), - outputTokens: formatMetric( - average(controls.map(result => result.assistant.outputTokens)), - average(baselines.map(result => result.assistant.outputTokens)), - value => numberFormatter.format(value), - ), - premiumRequests: formatMetric( - average(controls.map(result => result.assistant.premiumRequests)), - average(baselines.map(result => result.assistant.premiumRequests)), - value => numberFormatter.format(value), - ), - apiDuration: formatMetric( - average(controls.map(result => result.assistant.totalApiDurationMs)), - average(baselines.map(result => result.assistant.totalApiDurationMs)), - formatDuration, - ), - sessionDuration: formatMetric( - average(controls.map(result => result.assistant.sessionDurationMs)), - average(baselines.map(result => result.assistant.sessionDurationMs)), - formatDuration, - ), - toolCalls: formatMetric(average(controls.map(countToolCalls)), average(baselines.map(countToolCalls)), value => - numberFormatter.format(value), - ), - } - }) - }) - - return results - .toSorted((a, b) => { - if (a.passRate !== b.passRate) { - return (b.passRate ?? -1) - (a.passRate ?? -1) - } - - if (a.turnsValue !== b.turnsValue) { - return (a.turnsValue ?? Number.POSITIVE_INFINITY) - (b.turnsValue ?? Number.POSITIVE_INFINITY) - } - - return a.model.localeCompare(b.model) - }) - .map(result => ({ - id: result.id, - model: result.model, - reasoningEffort: result.reasoningEffort, - tests: result.tests, - turns: result.turns, - outputTokens: result.outputTokens, - premiumRequests: result.premiumRequests, - apiDuration: result.apiDuration, - sessionDuration: result.sessionDuration, - toolCalls: result.toolCalls, - })) -} - -function getBaselineTrendPoints(date: string, output: AgentEvalOutput): Array { - const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control') - const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended') - if (!controlTreatment || !baselineTreatment) { - return [] - } - - const controls = new Map( - output.results - .filter(result => result.treatmentId === controlTreatment.id) - .map(result => [getResultKey(result.scenarioId, result.model, result.reasoningEffort), result]), - ) - - return output.results - .filter(result => result.treatmentId === baselineTreatment.id) - .map(result => { - const control = controls.get(getResultKey(result.scenarioId, result.model, result.reasoningEffort)) - const controlToolCalls = countToolCalls(control) - const baselineToolCalls = countToolCalls(result) - const metric = ( - value: number, - controlValue: number | undefined, - raw = numberFormatter.format(value), - controlRaw = controlValue === undefined ? null : numberFormatter.format(controlValue), - ) => ({ - value, - raw, - change: getPercentChangeValue(controlValue, value), - controlValue: controlValue ?? null, - controlRaw, - }) - - return { - id: `${date}:${result.id}`, - date, - scenarioId: result.scenarioId, - model: result.model, - reasoningEffort: result.reasoningEffort ?? '—', - metrics: { - tests: { - value: result.testResults.numPassedTests, - raw: `${result.testResults.numPassedTests}/${result.testResults.numTotalTests}`, - change: getPercentChangeValue(control?.testResults.numPassedTests, result.testResults.numPassedTests), - controlValue: control?.testResults.numPassedTests ?? null, - controlRaw: control ? `${control.testResults.numPassedTests}/${control.testResults.numTotalTests}` : null, - }, - turns: metric(result.assistant.turns, control?.assistant.turns), - outputTokens: metric(result.assistant.outputTokens, control?.assistant.outputTokens), - premiumRequests: metric(result.assistant.premiumRequests, control?.assistant.premiumRequests), - apiDuration: metric( - result.assistant.totalApiDurationMs / 1000, - control ? control.assistant.totalApiDurationMs / 1000 : undefined, - formatDuration(result.assistant.totalApiDurationMs), - control ? formatDuration(control.assistant.totalApiDurationMs) : null, - ), - sessionDuration: metric( - result.assistant.sessionDurationMs / 1000, - control ? control.assistant.sessionDurationMs / 1000 : undefined, - formatDuration(result.assistant.sessionDurationMs), - control ? formatDuration(control.assistant.sessionDurationMs) : null, - ), - toolCalls: metric(baselineToolCalls ?? 0, controlToolCalls), - }, - } - }) -} - -function getAggregateTrendPoints(date: string, output: AgentEvalOutput): Array { - const controlTreatment = output.treatments.find(treatment => treatment.config.name === 'Control') - const baselineTreatment = output.treatments.find(treatment => treatment.config.name === 'Recommended') - if (!controlTreatment || !baselineTreatment) { - return [] - } - - return output.experiment.models.flatMap(model => { - const reasoningEfforts = model.reasoningEfforts.length > 0 ? model.reasoningEfforts : [undefined] - return reasoningEfforts.map(reasoningEffort => { - const matches = (result: AgentEvalOutputResult) => - result.model === model.name && result.reasoningEffort === reasoningEffort - const controls = output.results.filter(result => result.treatmentId === controlTreatment.id && matches(result)) - const baselines = output.results.filter(result => result.treatmentId === baselineTreatment.id && matches(result)) - const metric = ( - baselineValue: number | undefined, - controlValue: number | undefined, - raw = baselineValue === undefined ? '—' : numberFormatter.format(baselineValue), - controlRaw = controlValue === undefined ? null : numberFormatter.format(controlValue), - ) => ({ - value: baselineValue ?? null, - raw, - change: getPercentChangeValue(controlValue, baselineValue), - controlValue: controlValue ?? null, - controlRaw, - }) - const baselinePassedTests = average(baselines.map(result => result.testResults.numPassedTests)) - const baselineTotalTests = average(baselines.map(result => result.testResults.numTotalTests)) - const controlPassedTests = average(controls.map(result => result.testResults.numPassedTests)) - - return { - id: `${date}:aggregate:${model.name}:${reasoningEffort ?? ''}`, - date, - scenarioId: 'aggregate', - model: model.name, - reasoningEffort: reasoningEffort ?? '—', - metrics: { - tests: { - value: baselinePassedTests ?? null, - raw: - baselinePassedTests === undefined || baselineTotalTests === undefined - ? '—' - : `${numberFormatter.format(baselinePassedTests)}/${numberFormatter.format(baselineTotalTests)}`, - change: getPercentChangeValue(controlPassedTests, baselinePassedTests), - controlValue: controlPassedTests ?? null, - controlRaw: (() => { - const passed = average(controls.map(result => result.testResults.numPassedTests)) - const total = average(controls.map(result => result.testResults.numTotalTests)) - return passed === undefined || total === undefined - ? null - : `${numberFormatter.format(passed)}/${numberFormatter.format(total)}` - })(), - }, - turns: metric( - average(baselines.map(result => result.assistant.turns)), - average(controls.map(result => result.assistant.turns)), - ), - outputTokens: metric( - average(baselines.map(result => result.assistant.outputTokens)), - average(controls.map(result => result.assistant.outputTokens)), - ), - premiumRequests: metric( - average(baselines.map(result => result.assistant.premiumRequests)), - average(controls.map(result => result.assistant.premiumRequests)), - ), - apiDuration: (() => { - const baselineValue = average(baselines.map(result => result.assistant.totalApiDurationMs)) - const controlValue = average(controls.map(result => result.assistant.totalApiDurationMs)) - return metric( - baselineValue === undefined ? undefined : baselineValue / 1000, - controlValue === undefined ? undefined : controlValue / 1000, - baselineValue === undefined ? '—' : formatDuration(baselineValue), - controlValue === undefined ? null : formatDuration(controlValue), - ) - })(), - sessionDuration: (() => { - const baselineValue = average(baselines.map(result => result.assistant.sessionDurationMs)) - const controlValue = average(controls.map(result => result.assistant.sessionDurationMs)) - return metric( - baselineValue === undefined ? undefined : baselineValue / 1000, - controlValue === undefined ? undefined : controlValue / 1000, - baselineValue === undefined ? '—' : formatDuration(baselineValue), - controlValue === undefined ? null : formatDuration(controlValue), - ) - })(), - toolCalls: metric(average(baselines.map(countToolCalls)), average(controls.map(countToolCalls))), - }, - } - }) - }) -} - -async function getBaselinePageData() { - const runs = await listRuns() - const baselineRuns = runs.filter(run => run.output.experiment.id === 'baseline') - const latestRun = baselineRuns[0] - const baseline = latestRun ? getBaselineComparisons(latestRun.output) : null - const baselineTrends = baselineRuns.flatMap(run => getBaselineTrendPoints(run.name, run.output)) - const aggregateResults = latestRun ? getAggregateBaselineResults(latestRun.output) : [] - const aggregateTrends = baselineRuns.flatMap(run => getAggregateTrendPoints(run.name, run.output)) - - return {aggregateResults, aggregateTrends, baseline, baselineTrends} -} - -export {getBaselinePageData} diff --git a/website/src/benchmark-page-data.ts b/website/src/benchmark-page-data.ts new file mode 100644 index 00000000..60208817 --- /dev/null +++ b/website/src/benchmark-page-data.ts @@ -0,0 +1,28 @@ +import {get as getBenchmark} from './benchmarks' +import {getBenchmarkOverviewData, getBenchmarkPageResults, listBenchmarkRuns} from './benchmark-results' + +async function getBenchmarkPageData(id: string) { + const [benchmark, runs] = await Promise.all([getBenchmark(id), listBenchmarkRuns(id)]) + + return { + benchmark, + overview: getBenchmarkOverviewData(runs), + results: getBenchmarkPageResults(benchmark, runs[0]), + runs: runs.map(run => { + const trials = [...run.output.trials.values()] + return { + id: run.id, + name: run.name, + resultCount: trials.length, + passedTests: trials.reduce((total, trial) => { + return total + trial.testResults.numPassedTests + }, 0), + totalTests: trials.reduce((total, trial) => { + return total + trial.testResults.numTotalTests + }, 0), + } + }), + } +} + +export {getBenchmarkPageData} diff --git a/website/src/benchmark-results.ts b/website/src/benchmark-results.ts new file mode 100644 index 00000000..2dcc4dae --- /dev/null +++ b/website/src/benchmark-results.ts @@ -0,0 +1,598 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import type {BenchmarkOutput} from '@primer/agent-eval/benchmark' +import type {Benchmark} from './benchmarks' + +const {deserialize} = await import( + /* turbopackIgnore: true */ + '@primer/agent-eval/benchmark' +) + +const REPOSITORY_ROOT = path.resolve(process.cwd(), '..') +const BENCHMARK_RESULTS_DIR = path.join(REPOSITORY_ROOT, 'results', 'benchmarks') + +type BenchmarkOutputTrial = BenchmarkOutput['trials'] extends Map ? Trial : never + +type ResultTotals = { + passed: number + total: number + outputTokens: number + premiumRequests: number + sessionDurationMs: number + totalApiDurationMs: number +} + +export type BenchmarkComparison = { + tests: string + outputTokens: string + premiumRequests: string + sessionTime: string + apiTime: string +} + +export type BenchmarkCapabilityResult = { + id: string + name: string + comparison: BenchmarkComparison + scenarios: Array<{ + id: string + comparison: BenchmarkComparison + models: Array<{ + id: string + name: string + reasoningEffort: string + comparison: BenchmarkComparison + }> + }> +} + +export type BenchmarkPageResults = { + date: string + capabilities: Array +} + +export type BenchmarkOverviewResult = { + id: string + model: string + reasoningEffort: string + comparison: BenchmarkComparison +} + +export type BenchmarkTrendMetricId = 'tests' | 'outputTokens' | 'premiumRequests' | 'sessionTime' | 'apiTime' + +export type BenchmarkTrendMetric = { + value: number | null + raw: string + change: number | null + controlValue: number | null + controlRaw: string | null +} + +export type BenchmarkTrendPoint = { + id: string + date: string + capabilityId: string | null + scenarioId: string | null + model: string + reasoningEffort: string + metrics: Record +} + +export type BenchmarkOverviewData = { + date: string | null + results: Array + trends: Array +} + +type OutputCandidate = { + filepath: string + date: string +} + +export type BenchmarkRun = { + id: string + name: string + directory: string + date: Date + output: BenchmarkOutput +} + +function isRunDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { + return false + } + + const date = new Date(`${value}T00:00:00.000Z`) + return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(value) +} + +async function getDatedCandidates(benchmarkId: string): Promise> { + const directory = path.join(BENCHMARK_RESULTS_DIR, benchmarkId) + + try { + const entries = await fs.readdir(directory, {withFileTypes: true}) + const candidates = await Promise.all( + entries + .filter(entry => { + return entry.isDirectory() && isRunDate(entry.name) + }) + .map(async entry => { + const filepath = path.join(directory, entry.name, 'output.json') + try { + await fs.access(filepath) + return { + filepath, + date: entry.name, + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return null + } + + throw error + } + }), + ) + + return candidates.filter((candidate): candidate is OutputCandidate => { + return candidate !== null + }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return [] + } + + throw error + } +} + +async function readBenchmarkOutput(candidate: OutputCandidate, benchmarkId: string): Promise { + const contents = await fs.readFile(candidate.filepath, 'utf-8') + const parsed: unknown = JSON.parse(contents) + const output = deserialize(parsed) + return output.benchmarkId === benchmarkId ? output : null +} + +function getTotals(trials: Array): ResultTotals { + return trials.reduce( + (totals, trial) => { + totals.passed += trial.testResults.numPassedTests + totals.total += trial.testResults.numTotalTests + for (const session of trial.agent.sessions) { + totals.outputTokens += session.outputTokens + totals.premiumRequests += session.premiumRequests + totals.sessionDurationMs += session.sessionDurationMs + totals.totalApiDurationMs += session.totalApiDurationMs + } + return totals + }, + { + passed: 0, + total: 0, + outputTokens: 0, + premiumRequests: 0, + sessionDurationMs: 0, + totalApiDurationMs: 0, + }, + ) +} + +function getPassRate(totals: ResultTotals): number | null { + if (totals.total === 0) { + return null + } + + return totals.passed / totals.total +} + +function getPercentDelta(control: number, benchmark: number): number | null { + if (control === 0) { + return benchmark === 0 ? 0 : null + } + + return (benchmark - control) / control +} + +function getPercentDeltaValue(control: number, benchmark: number): number | null { + const delta = getPercentDelta(control, benchmark) + return delta === null ? null : delta * 100 +} + +function formatPercentDelta(control: number, benchmark: number): string { + if (control === benchmark) { + return '0%' + } + + const delta = getPercentDelta(control, benchmark) + if (delta === null) { + return 'N/A' + } + + const sign = delta > 0 ? '+' : '' + return `${sign}${(delta * 100).toFixed(1)}%` +} + +function formatValue(value: string, control: number, benchmark: number): string { + return `${value} (${formatPercentDelta(control, benchmark)})` +} + +function formatNumber(value: number): string { + return new Intl.NumberFormat('en-US').format(value) +} + +function formatDuration(milliseconds: number): string { + const seconds = milliseconds / 1000 + if (seconds < 60) { + return `${seconds.toFixed(1)}s` + } + + const minutes = Math.floor(seconds / 60) + const remainingSeconds = seconds - minutes * 60 + return `${minutes}m ${remainingSeconds.toFixed(1)}s` +} + +function createComparison( + trials: Array, + controlTreatmentId: string, + benchmarkTreatmentId: string, +): BenchmarkComparison { + const controlTotals = getTotals( + trials.filter(trial => { + return trial.treatmentId === controlTreatmentId + }), + ) + const benchmarkTotals = getTotals( + trials.filter(trial => { + return trial.treatmentId === benchmarkTreatmentId + }), + ) + const controlPassRate = getPassRate(controlTotals) ?? 0 + const benchmarkPassRate = getPassRate(benchmarkTotals) ?? 0 + + return { + tests: formatValue(`${benchmarkTotals.passed}/${benchmarkTotals.total}`, controlPassRate, benchmarkPassRate), + outputTokens: formatValue( + formatNumber(benchmarkTotals.outputTokens), + controlTotals.outputTokens, + benchmarkTotals.outputTokens, + ), + premiumRequests: formatValue( + formatNumber(benchmarkTotals.premiumRequests), + controlTotals.premiumRequests, + benchmarkTotals.premiumRequests, + ), + sessionTime: formatValue( + formatDuration(benchmarkTotals.sessionDurationMs), + controlTotals.sessionDurationMs, + benchmarkTotals.sessionDurationMs, + ), + apiTime: formatValue( + formatDuration(benchmarkTotals.totalApiDurationMs), + controlTotals.totalApiDurationMs, + benchmarkTotals.totalApiDurationMs, + ), + } +} + +function getTreatments(output: BenchmarkOutput): { + controlTreatmentId: string + benchmarkTreatmentId: string +} { + const controlTreatment = [...output.treatments].find(([, treatment]) => { + return treatment.name === 'Control' + }) + const benchmarkTreatment = [...output.treatments].find(([, treatment]) => { + return treatment.name === 'Benchmark' + }) + + if (!controlTreatment || !benchmarkTreatment) { + throw new Error(`Benchmark "${output.benchmarkId}" results must include Control and Benchmark treatments`) + } + + return { + controlTreatmentId: controlTreatment[0], + benchmarkTreatmentId: benchmarkTreatment[0], + } +} + +function groupTrialsByModel(trials: Array): Array> { + const modelTrials = new Map>() + for (const trial of trials) { + const key = `${trial.model.name}\0${trial.model.reasoningEffort}` + const trialsForModel = modelTrials.get(key) ?? [] + trialsForModel.push(trial) + modelTrials.set(key, trialsForModel) + } + return [...modelTrials.values()] +} + +function createTrendMetric( + value: number | null, + raw: string, + controlValue: number | null, + controlRaw: string | null, + controlDeltaValue: number, + benchmarkDeltaValue: number, +): BenchmarkTrendMetric { + return { + value, + raw, + change: getPercentDeltaValue(controlDeltaValue, benchmarkDeltaValue), + controlValue, + controlRaw, + } +} + +function createTrendPoint( + date: string, + trials: Array, + controlTreatmentId: string, + benchmarkTreatmentId: string, + capabilityId: string | null, + scenarioId: string | null, +): BenchmarkTrendPoint { + const trial = trials[0] + const controlTotals = getTotals( + trials.filter(candidate => { + return candidate.treatmentId === controlTreatmentId + }), + ) + const benchmarkTotals = getTotals( + trials.filter(candidate => { + return candidate.treatmentId === benchmarkTreatmentId + }), + ) + const controlPassRate = getPassRate(controlTotals) + const benchmarkPassRate = getPassRate(benchmarkTotals) + + return { + id: `${date}:${capabilityId ?? 'all'}:${scenarioId ?? 'all'}:${trial.model.name}:${trial.model.reasoningEffort}`, + date, + capabilityId, + scenarioId, + model: trial.model.name, + reasoningEffort: trial.model.reasoningEffort, + metrics: { + tests: createTrendMetric( + benchmarkPassRate === null ? null : benchmarkPassRate * 100, + `${benchmarkTotals.passed}/${benchmarkTotals.total}`, + controlPassRate === null ? null : controlPassRate * 100, + `${controlTotals.passed}/${controlTotals.total}`, + controlPassRate ?? 0, + benchmarkPassRate ?? 0, + ), + outputTokens: createTrendMetric( + benchmarkTotals.outputTokens, + formatNumber(benchmarkTotals.outputTokens), + controlTotals.outputTokens, + formatNumber(controlTotals.outputTokens), + controlTotals.outputTokens, + benchmarkTotals.outputTokens, + ), + premiumRequests: createTrendMetric( + benchmarkTotals.premiumRequests, + formatNumber(benchmarkTotals.premiumRequests), + controlTotals.premiumRequests, + formatNumber(controlTotals.premiumRequests), + controlTotals.premiumRequests, + benchmarkTotals.premiumRequests, + ), + sessionTime: createTrendMetric( + benchmarkTotals.sessionDurationMs / 1000, + formatDuration(benchmarkTotals.sessionDurationMs), + controlTotals.sessionDurationMs / 1000, + formatDuration(controlTotals.sessionDurationMs), + controlTotals.sessionDurationMs, + benchmarkTotals.sessionDurationMs, + ), + apiTime: createTrendMetric( + benchmarkTotals.totalApiDurationMs / 1000, + formatDuration(benchmarkTotals.totalApiDurationMs), + controlTotals.totalApiDurationMs / 1000, + formatDuration(controlTotals.totalApiDurationMs), + controlTotals.totalApiDurationMs, + benchmarkTotals.totalApiDurationMs, + ), + }, + } +} + +function createTrendPoints( + date: string, + trials: Array, + controlTreatmentId: string, + benchmarkTreatmentId: string, + capabilityId: string | null, + scenarioId: string | null, +): Array { + return groupTrialsByModel(trials).map(modelTrials => { + return createTrendPoint(date, modelTrials, controlTreatmentId, benchmarkTreatmentId, capabilityId, scenarioId) + }) +} + +function compareModelPerformance( + a: Array, + b: Array, + benchmarkTreatmentId: string, +): number { + const aTotals = getTotals( + a.filter(trial => { + return trial.treatmentId === benchmarkTreatmentId + }), + ) + const bTotals = getTotals( + b.filter(trial => { + return trial.treatmentId === benchmarkTreatmentId + }), + ) + + return ( + (getPassRate(bTotals) ?? 0) - (getPassRate(aTotals) ?? 0) || + aTotals.outputTokens - bTotals.outputTokens || + aTotals.premiumRequests - bTotals.premiumRequests || + aTotals.sessionDurationMs - bTotals.sessionDurationMs || + aTotals.totalApiDurationMs - bTotals.totalApiDurationMs + ) +} + +function createPageResults(benchmark: Benchmark, output: BenchmarkOutput, date: string): BenchmarkPageResults { + const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(output) + + const trials = [...output.trials.values()] + const capabilities = benchmark.capabilities.map((capability, capabilityIndex) => { + const capabilityTrials = trials.filter(trial => { + return trial.capabilityId === capability.name + }) + + return { + id: `${capabilityIndex}-${capability.name}`, + name: capability.name, + comparison: createComparison(capabilityTrials, controlTreatmentId, benchmarkTreatmentId), + scenarios: capability.scenarios.map(scenario => { + const scenarioTrials = capabilityTrials.filter(trial => { + return trial.scenarioId === scenario.id + }) + return { + id: scenario.id, + comparison: createComparison(scenarioTrials, controlTreatmentId, benchmarkTreatmentId), + models: groupTrialsByModel(scenarioTrials) + .toSorted((a, b) => { + return compareModelPerformance(a, b, benchmarkTreatmentId) + }) + .map(trialsForModel => { + const trial = trialsForModel[0] + return { + id: `${trial.model.name}\0${trial.model.reasoningEffort}`, + name: trial.model.name, + reasoningEffort: trial.model.reasoningEffort, + comparison: createComparison(trialsForModel, controlTreatmentId, benchmarkTreatmentId), + } + }), + } + }), + } + }) + + return {date, capabilities} +} + +export function getBenchmarkOverviewData(runs: Array): BenchmarkOverviewData { + const latestRun = runs[0] + const results = latestRun + ? (() => { + const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(latestRun.output) + return groupTrialsByModel([...latestRun.output.trials.values()]) + .toSorted((a, b) => { + return compareModelPerformance(a, b, benchmarkTreatmentId) + }) + .map(trials => { + const trial = trials[0] + return { + id: `${trial.model.name}\0${trial.model.reasoningEffort}`, + model: trial.model.name, + reasoningEffort: trial.model.reasoningEffort, + comparison: createComparison(trials, controlTreatmentId, benchmarkTreatmentId), + } + }) + })() + : [] + const trends = runs.flatMap(run => { + const {controlTreatmentId, benchmarkTreatmentId} = getTreatments(run.output) + const trials = [...run.output.trials.values()] + const scenarioIds = new Set( + trials.map(trial => { + return trial.scenarioId + }), + ) + const allTrendPoints = createTrendPoints(run.name, trials, controlTreatmentId, benchmarkTreatmentId, null, null) + const scenarioTrendPoints = [...scenarioIds].flatMap(scenarioId => { + return createTrendPoints( + run.name, + trials.filter(trial => { + return trial.scenarioId === scenarioId + }), + controlTreatmentId, + benchmarkTreatmentId, + null, + scenarioId, + ) + }) + const capabilityTrendPoints = [...run.output.capabilities.values()].flatMap(capability => { + const capabilityTrials = trials.filter(trial => { + return trial.capabilityId === capability.name + }) + const capabilityPoints = createTrendPoints( + run.name, + capabilityTrials, + controlTreatmentId, + benchmarkTreatmentId, + capability.name, + null, + ) + const capabilityScenarioPoints = capability.scenarioIds.flatMap(scenarioId => { + return createTrendPoints( + run.name, + capabilityTrials.filter(trial => { + return trial.scenarioId === scenarioId + }), + controlTreatmentId, + benchmarkTreatmentId, + capability.name, + scenarioId, + ) + }) + + return [...capabilityPoints, ...capabilityScenarioPoints] + }) + + return [...allTrendPoints, ...scenarioTrendPoints, ...capabilityTrendPoints] + }) + + return { + date: latestRun?.name ?? null, + results, + trends, + } +} + +export async function listBenchmarkRuns(benchmarkId: string): Promise> { + const candidates = (await getDatedCandidates(benchmarkId)).toSorted((a, b) => { + return b.date.localeCompare(a.date) + }) + + const runs: Array = [] + for (const candidate of candidates) { + const output = await readBenchmarkOutput(candidate, benchmarkId) + if (output) { + runs.push({ + id: candidate.date, + name: candidate.date, + directory: path.dirname(candidate.filepath), + date: new Date(`${candidate.date}T00:00:00.000Z`), + output, + }) + } + } + + return runs +} + +export async function getBenchmarkRun(benchmarkId: string, date: string): Promise { + if (!isRunDate(date)) { + return null + } + + const runs = await listBenchmarkRuns(benchmarkId) + return ( + runs.find(run => { + return run.name === date + }) ?? null + ) +} + +export function getBenchmarkPageResults( + benchmark: Benchmark, + run: BenchmarkRun | undefined, +): BenchmarkPageResults | null { + return run ? createPageResults(benchmark, run.output, run.name) : null +} diff --git a/website/src/benchmarks.ts b/website/src/benchmarks.ts new file mode 100644 index 00000000..bf6b45e0 --- /dev/null +++ b/website/src/benchmarks.ts @@ -0,0 +1,53 @@ +import path from 'node:path' +import type {Benchmark as AgentEvalBenchmark} from '@primer/agent-eval/benchmark' + +const {getBenchmark, listBenchmarks} = await import( + /* turbopackIgnore: true */ + '@primer/agent-eval/benchmark' +) + +const BENCHMARKS_DIR = path.resolve(process.cwd(), '..', 'benchmarks') +const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios') + +export type Benchmark = Pick & { + capabilities: Array<{ + name: string + scenarios: Array<{id: string}> + }> +} + +function normalizeBenchmark(benchmark: AgentEvalBenchmark): Benchmark { + return { + id: benchmark.id, + name: benchmark.name, + description: benchmark.description, + models: benchmark.models, + capabilities: benchmark.capabilities.map(capability => { + return { + name: capability.name, + scenarios: capability.scenarios.map(scenario => { + return {id: scenario.id} + }), + } + }), + } +} + +export async function list(): Promise> { + const benchmarks = await listBenchmarks({ + benchmarksDirectory: BENCHMARKS_DIR, + scenariosDirectory: SCENARIOS_DIR, + }) + + return benchmarks.map(normalizeBenchmark) +} + +export async function get(id: string): Promise { + const benchmark = await getBenchmark({ + benchmarksDirectory: BENCHMARKS_DIR, + scenariosDirectory: SCENARIOS_DIR, + id, + }) + + return normalizeBenchmark(benchmark) +} diff --git a/website/src/experiments.ts b/website/src/experiments.ts index fe7a37cc..1c386a06 100644 --- a/website/src/experiments.ts +++ b/website/src/experiments.ts @@ -1,50 +1,50 @@ import path from 'node:path' -import type {ExperimentConfig} from '@primer/agent-eval/experiment' +import type {Experiment as AgentEvalExperiment} from '@primer/agent-eval/experiment' -const {listExperiments, findExperiment} = await import( +const {listExperiments, getExperiment} = await import( /* turbopackIgnore: true */ - '@primer/agent-eval/experiments' + '@primer/agent-eval/experiment' ) const EXPERIMENTS_DIR = path.resolve(process.cwd(), '..', 'experiments') +const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios') -export type Experiment = Pick & { - id: string +export type Experiment = Pick & { + scenarios: Array<{id: string}> treatments: Array<{name: string}> } export async function list(): Promise> { const experiments = await listExperiments({ - directory: EXPERIMENTS_DIR, + experimentsDirectory: EXPERIMENTS_DIR, + scenariosDirectory: SCENARIOS_DIR, }) - return experiments.map(([id, experiment]) => { + return experiments.map(experiment => { return { - id, + id: experiment.id, name: experiment.name, description: experiment.description, models: experiment.models, - scenarios: experiment.scenarios, + scenarios: experiment.scenarios.map(scenario => ({id: scenario.id})), treatments: experiment.treatments.map(t => ({name: t.name})), } }) } export async function get(id: string): Promise { - const experiment = await findExperiment(id, { - directory: EXPERIMENTS_DIR, + const experiment = await getExperiment({ + experimentsDirectory: EXPERIMENTS_DIR, + scenariosDirectory: SCENARIOS_DIR, + id, }) - if (!experiment) { - throw new Error(`Experiment "${id}" was not found in: ${EXPERIMENTS_DIR}`) - } - return { - id, + id: experiment.id, name: experiment.name, description: experiment.description, models: experiment.models, - scenarios: experiment.scenarios, + scenarios: experiment.scenarios.map(scenario => ({id: scenario.id})), treatments: experiment.treatments.map(t => ({name: t.name})), } } @@ -52,13 +52,5 @@ export async function get(id: string): Promise { export async function listForScenario(id: string): Promise> { const experiments = await list() - return experiments.filter(experiment => - experiment.scenarios.some(scenario => { - if (typeof scenario === 'string') { - return scenario === id - } - - return (scenario.name ?? path.basename(path.resolve(scenario.path))) === id - }), - ) + return experiments.filter(experiment => experiment.scenarios.some(scenario => scenario.id === id)) } diff --git a/website/src/run-details.ts b/website/src/run-details.ts new file mode 100644 index 00000000..330067c2 --- /dev/null +++ b/website/src/run-details.ts @@ -0,0 +1,335 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import type {RunOutput, RunOutputResult} from './runs' + +const REPOSITORY_ROOT = path.resolve(process.cwd(), '..') +const LEGACY_ARTIFACTS_DIRECTORY = path.join(REPOSITORY_ROOT, 'artifacts') + +type LogMessage = RunOutputResult['assistant']['logs'][number] +type Walkthrough = RunOutputResult['walkthrough'] + +type TranscriptEntry = { + id: string + label: string + timestamp?: string + content: string +} + +type WalkthroughDataUrl = + | { + type: 'Unavailable' + } + | {type: 'Screenshot'; screenshot: string} + | {type: 'Screenshots'; screenshots: Array} + | {type: 'Video'; video: string} + +type RunResult = { + id: string + scenarioId: string + context?: string + treatment: string + model: string + reasoningEffort?: string + testsPassed: number + totalTests: number + turns: number + outputTokens: number + premiumRequests: number + totalApiDurationMs: number + sessionDurationMs: number + tests: Array<{ + fullName: string + status: string + description?: string + }> + transcript: Array + walkthrough: WalkthroughDataUrl +} + +type RunDetails = { + date: string + results: Array +} + +function asRecord(value: unknown): Record | null { + return typeof value === 'object' && value !== null ? (value as Record) : null +} + +function getString(record: Record | null, key: string): string | undefined { + const value = record?.[key] + return typeof value === 'string' ? value : undefined +} + +function createTranscript(logs: Array): Array { + const entries: Array = [] + const messageEntries = new Map() + const reasoningEntries = new Map() + const toolNames = new Map() + + for (const [index, message] of logs.entries()) { + const record = asRecord(message) + const data = asRecord(record?.data) + const timestamp = getString(record, 'timestamp') + const id = getString(record, 'id') ?? `${message.type}-${index}` + + switch (message.type) { + case 'user.message': { + const content = getString(data, 'content') + if (content) { + entries.push({id, label: 'User', timestamp, content}) + } + break + } + case 'assistant.message_delta': { + const messageId = getString(data, 'messageId') + const delta = getString(data, 'deltaContent') + if (!messageId || !delta) { + break + } + + let entry = messageEntries.get(messageId) + if (!entry) { + entry = {id, label: 'Assistant', timestamp, content: ''} + messageEntries.set(messageId, entry) + entries.push(entry) + } + entry.content += delta + break + } + case 'assistant.message': { + const messageId = getString(data, 'messageId') + const content = getString(data, 'content') + const entry = messageId ? messageEntries.get(messageId) : undefined + if (entry) { + if (content) { + entry.content = content + } + } else if (content) { + entries.push({id, label: 'Assistant', timestamp, content}) + } + break + } + case 'assistant.reasoning_delta': { + const reasoningId = getString(data, 'reasoningId') + const delta = getString(data, 'deltaContent') + if (!reasoningId || !delta) { + break + } + + let entry = reasoningEntries.get(reasoningId) + if (!entry) { + entry = {id, label: 'Reasoning', timestamp, content: ''} + reasoningEntries.set(reasoningId, entry) + entries.push(entry) + } + entry.content += delta + break + } + case 'assistant.reasoning': { + const reasoningId = getString(data, 'reasoningId') + const content = getString(data, 'content') + const entry = reasoningId ? reasoningEntries.get(reasoningId) : undefined + if (entry) { + if (content) { + entry.content = content + } + } else if (content) { + entries.push({id, label: 'Reasoning', timestamp, content}) + } + break + } + case 'tool.execution_start': { + const toolName = getString(data, 'toolName') ?? 'Unknown tool' + const toolCallId = getString(data, 'toolCallId') + if (toolCallId) { + toolNames.set(toolCallId, toolName) + } + entries.push({id, label: `Tool call: ${toolName}`, timestamp, content: 'Started'}) + break + } + case 'tool.execution_complete': { + const toolCallId = getString(data, 'toolCallId') + const toolName = toolCallId ? toolNames.get(toolCallId) : undefined + entries.push({ + id, + label: `Tool result: ${toolName ?? 'Unknown tool'}`, + timestamp, + content: data?.success === true ? 'Completed successfully' : 'Failed', + }) + break + } + case 'session.info': { + const content = getString(data, 'message') + if (content) { + entries.push({id, label: 'Session', timestamp, content}) + } + break + } + case 'session.task_complete': { + const content = getString(data, 'summary') + if (content) { + entries.push({id, label: 'Summary', timestamp, content}) + } + break + } + } + } + + return entries.filter(entry => { + return entry.content.length > 0 + }) +} + +function isWithinDirectory(directory: string, filepath: string): boolean { + const relativePath = path.relative(directory, filepath) + return relativePath !== '..' && !relativePath.startsWith(`..${path.sep}`) && !path.isAbsolute(relativePath) +} + +function getArtifactCandidates(artifactPath: string, runDirectory: string): Array { + const runArtifactsDirectory = path.join(runDirectory, 'artifacts') + + if (!path.isAbsolute(artifactPath)) { + const candidate = path.resolve(runDirectory, artifactPath) + return isWithinDirectory(runArtifactsDirectory, candidate) ? [candidate] : [] + } + + if (isWithinDirectory(LEGACY_ARTIFACTS_DIRECTORY, artifactPath)) { + return [artifactPath] + } + + const segments = artifactPath.split(/[\\/]+/) + const artifactsIndex = segments.lastIndexOf('artifacts') + if (artifactsIndex === -1) { + return [] + } + + const artifactSegments = segments.slice(artifactsIndex + 1) + return [ + path.join(runArtifactsDirectory, ...artifactSegments), + path.join(LEGACY_ARTIFACTS_DIRECTORY, ...artifactSegments), + ].filter(candidate => { + return ( + isWithinDirectory(runArtifactsDirectory, candidate) || isWithinDirectory(LEGACY_ARTIFACTS_DIRECTORY, candidate) + ) + }) +} + +async function getArtifactDataUrl( + artifactPath: string | undefined, + mimeType: string, + runDirectory: string, +): Promise { + if (!artifactPath) { + return undefined + } + + for (const candidate of getArtifactCandidates(artifactPath, runDirectory)) { + try { + const contents = await fs.readFile(candidate) + return `data:${mimeType};base64,${contents.toString('base64')}` + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + } + + return undefined +} + +function getImageMimeType(artifactPath: string): string { + const extension = path.extname(artifactPath).toLowerCase() + if (extension === '.jpg' || extension === '.jpeg') { + return 'image/jpeg' + } + return 'image/png' +} + +async function getWalkthroughDataUrls(walkthrough: Walkthrough, runDirectory: string): Promise { + if (walkthrough.type === 'Screenshot') { + const screenshot = await getArtifactDataUrl( + walkthrough.filepath, + getImageMimeType(walkthrough.filepath), + runDirectory, + ) + return screenshot + ? { + type: 'Screenshot', + screenshot, + } + : {type: 'Unavailable'} + } + + if (walkthrough.type === 'Screenshots') { + const sources = await Promise.all( + walkthrough.screenshots.map(artifactPath => { + return getArtifactDataUrl(artifactPath, getImageMimeType(artifactPath), runDirectory) + }), + ) + const screenshots = sources.filter((source): source is string => { + return source !== undefined + }) + return screenshots.length > 0 + ? { + type: 'Screenshots', + screenshots, + } + : {type: 'Unavailable'} + } + + if (walkthrough.type === 'Video') { + const video = await getArtifactDataUrl(walkthrough.filepath, 'video/webm', runDirectory) + return video + ? { + type: 'Video', + video, + } + : {type: 'Unavailable'} + } + + return { + type: 'Unavailable', + } +} + +async function createExperimentRunDetails(date: string, output: RunOutput, runDirectory: string): Promise { + const treatments = new Map( + output.treatments.map(treatment => { + return [treatment.id, treatment.config.name] + }), + ) + + return { + date, + results: await Promise.all( + output.results.map(async result => { + return { + id: result.id, + scenarioId: result.scenarioId, + treatment: treatments.get(result.treatmentId) ?? 'Unknown treatment', + model: result.model, + reasoningEffort: result.reasoningEffort, + testsPassed: result.testResults.numPassedTests, + totalTests: result.testResults.numTotalTests, + turns: result.assistant.turns, + outputTokens: result.assistant.outputTokens, + premiumRequests: result.assistant.premiumRequests, + totalApiDurationMs: result.assistant.totalApiDurationMs, + sessionDurationMs: result.assistant.sessionDurationMs, + tests: result.testResults.tests.map(test => { + return { + fullName: test.fullName, + status: test.status, + description: test.description, + } + }), + walkthrough: await getWalkthroughDataUrls(result.walkthrough, runDirectory), + transcript: createTranscript(result.assistant.logs), + } + }), + ), + } +} + +export {createExperimentRunDetails, createTranscript, getWalkthroughDataUrls} +export type {RunDetails, TranscriptEntry, WalkthroughDataUrl} diff --git a/website/src/runs.ts b/website/src/runs.ts index 2da7a8d7..6e25417d 100644 --- a/website/src/runs.ts +++ b/website/src/runs.ts @@ -2,16 +2,103 @@ import fs from 'node:fs/promises' import {existsSync, type Dirent} from 'node:fs' import path from 'node:path' -import {parseAgentEvalOutput, type AgentEvalOutput} from '@primer/agent-eval/output' +import type {ExperimentOutput} from '@primer/agent-eval/experiment' -const RESULTS_DIR = path.resolve(process.cwd(), '..', 'results') +const {deserialize} = await import( + /* turbopackIgnore: true */ + '@primer/agent-eval/experiment' +) + +const RESULTS_DIR = path.resolve(process.cwd(), '..', 'results', 'experiments') + +type ExperimentOutputTrial = ExperimentOutput['trials'] extends Map ? Trial : never + +type RunOutputResult = { + id: string + treatmentId: string + model: ExperimentOutputTrial['model']['name'] + reasoningEffort: ExperimentOutputTrial['model']['reasoningEffort'] + scenarioId: string + assistant: { + logs: ExperimentOutputTrial['agent']['sessions'][number]['messages'] + turns: number + outputTokens: number + premiumRequests: number + totalApiDurationMs: number + sessionDurationMs: number + tools: Record + } + testResults: ExperimentOutputTrial['testResults'] & { + tests: Array<{ + title: string + fullName: string + status: string + description?: string + }> + } + walkthrough: ExperimentOutputTrial['walkthrough'] +} + +type RunOutput = { + experiment: { + id: string + models: Array<{ + name: ExperimentOutputTrial['model']['name'] + reasoningEfforts: Array + }> + } + scenarios: Array ? Scenario : never> + treatments: Array<{ + id: string + config: { + name: string + } + }> + results: Array +} type Run = { id: string name: string directory: string date: Date - output: AgentEvalOutput + output: RunOutput +} + +function asRecord(value: unknown): Record | null { + if (typeof value !== 'object' || value === null) { + return null + } + + return value as Record +} + +function isLegacyRunOutput(value: unknown): value is RunOutput { + const output = asRecord(value) + const experiment = asRecord(output?.experiment) + + return ( + typeof experiment?.id === 'string' && + Array.isArray(experiment.models) && + Array.isArray(output?.scenarios) && + Array.isArray(output.treatments) && + Array.isArray(output.results) + ) +} + +function parseOutput(contents: string): RunOutput { + const parsed: unknown = JSON.parse(contents) + const output = asRecord(parsed) + + if (typeof output?.experimentId === 'string') { + return normalizeOutput(deserialize(parsed)) + } + + if (isLegacyRunOutput(parsed)) { + return parsed + } + + throw new Error('Result output does not match a supported experiment output format') } function isRunName(name: string): boolean { @@ -23,10 +110,12 @@ function isRunName(name: string): boolean { return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(name) } -async function listResultDirectories(): Promise> { +async function listExperimentDirectories(): Promise> { try { const entries = await fs.readdir(RESULTS_DIR, {withFileTypes: true}) - return entries.filter(entry => entry.isDirectory() && isRunName(entry.name)) + return entries.filter(entry => { + return entry.isDirectory() + }) } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return [] @@ -36,51 +125,57 @@ async function listResultDirectories(): Promise> { } } -async function list(): Promise> { - const results = await listResultDirectories().then(entries => { - return Promise.all( - entries.map(async entry => { - const directory = path.join(RESULTS_DIR, entry.name) - const outputFile = path.join(directory, 'output.json') - const contents = await fs.readFile(outputFile, 'utf-8') - const output = parseAgentEvalOutput(contents) - return [directory, entry.name, output] as const - }), - ) - }) - - return results - .map(([directory, name, output]) => { - const date = new Date(`${name}T00:00:00.000Z`) - return {id: output.id, name, directory, date, output} +async function listRunDirectories(experimentId: string): Promise> { + const experimentDirectory = path.join(RESULTS_DIR, experimentId) + try { + const entries = await fs.readdir(experimentDirectory, {withFileTypes: true}) + return entries.filter(entry => { + return entry.isDirectory() && isRunName(entry.name) }) - .toSorted((a, b) => b.date.getTime() - a.date.getTime()) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return [] + } + + throw error + } } async function listForExperiment(experimentId: string): Promise> { - const runs = await list() - return runs.filter(run => run.output.experiment.id === experimentId) + const entries = await listRunDirectories(experimentId) + const runs = await Promise.all( + entries.map(entry => { + return find(experimentId, entry.name) + }), + ) + + return runs + .filter((run): run is Run => { + return run !== null + }) + .toSorted((a, b) => { + return b.date.getTime() - a.date.getTime() + }) } -async function latest(): Promise { - const runs = await listResultDirectories().then(entries => { - return entries.map(entry => [new Date(`${entry.name}T00:00:00.000Z`), entry.name] as const) +async function list(): Promise> { + const experiments = await listExperimentDirectories() + const runs = await Promise.all( + experiments.map(experiment => { + return listForExperiment(experiment.name) + }), + ) + return runs.flat().toSorted((a, b) => { + return b.date.getTime() - a.date.getTime() }) - if (runs.length === 0) { - return null - } - - const sorted = runs.toSorted((a, b) => b[0].getTime() - a[0].getTime()) - const run = await find(sorted[0][1]) - return run } -async function find(name: string): Promise { +async function find(experimentId: string, name: string): Promise { if (!isRunName(name)) { return null } - const directory = path.join(RESULTS_DIR, name) + const directory = path.join(RESULTS_DIR, experimentId, name) if (!existsSync(directory)) { return null } @@ -96,9 +191,13 @@ async function find(name: string): Promise { const outputFile = path.join(directory, 'output.json') const contents = await fs.readFile(outputFile, 'utf-8') - const output = parseAgentEvalOutput(contents) + const output = parseOutput(contents) + if (output.experiment.id !== experimentId) { + return null + } + return { - id: output.id, + id: name, name, directory, date: new Date(`${name}T00:00:00.000Z`), @@ -106,14 +205,87 @@ async function find(name: string): Promise { } } -async function get(name: string): Promise { - const run = await find(name) +async function get(experimentId: string, name: string): Promise { + const run = await find(experimentId, name) if (!run) { - throw new Error(`Run "${name}" was not found in: ${RESULTS_DIR}`) + throw new Error(`Run "${name}" for experiment "${experimentId}" was not found in: ${RESULTS_DIR}`) } return run } -export {list, listForExperiment, latest, get} -export type {Run} +function normalizeOutput(output: ExperimentOutput): RunOutput { + const modelReasoningEfforts = new Map< + ExperimentOutputTrial['model']['name'], + Set + >() + + const results = [...output.trials.values()].map(trial => { + const reasoningEfforts = modelReasoningEfforts.get(trial.model.name) ?? new Set() + reasoningEfforts.add(trial.model.reasoningEffort) + modelReasoningEfforts.set(trial.model.name, reasoningEfforts) + + const tools: Record = {} + for (const session of trial.agent.sessions) { + for (const [name, count] of Object.entries(session.tools)) { + tools[name] = (tools[name] ?? 0) + count + } + } + + return { + id: trial.id, + treatmentId: trial.treatmentId, + model: trial.model.name, + reasoningEffort: trial.model.reasoningEffort, + scenarioId: trial.scenarioId, + assistant: { + logs: trial.agent.sessions.flatMap(session => session.messages), + turns: trial.agent.sessions.reduce((total, session) => total + session.turns, 0), + outputTokens: trial.agent.sessions.reduce((total, session) => total + session.outputTokens, 0), + premiumRequests: trial.agent.sessions.reduce((total, session) => total + session.premiumRequests, 0), + totalApiDurationMs: trial.agent.sessions.reduce((total, session) => total + session.totalApiDurationMs, 0), + sessionDurationMs: trial.agent.sessions.reduce((total, session) => total + session.sessionDurationMs, 0), + tools, + }, + testResults: { + ...trial.testResults, + tests: trial.testResults.testResults.flatMap(testResult => { + return testResult.assertionResults.map(assertion => { + return { + title: assertion.title, + fullName: assertion.fullName, + status: assertion.status, + description: assertion.meta.description, + } + }) + }), + }, + walkthrough: trial.walkthrough, + } + }) + + return { + experiment: { + id: output.experimentId, + models: [...modelReasoningEfforts].map(([name, reasoningEfforts]) => { + return { + name, + reasoningEfforts: [...reasoningEfforts], + } + }), + }, + scenarios: [...output.scenarios.values()], + treatments: [...output.treatments].map(([id, treatment]) => { + return { + id, + config: { + name: treatment.name, + }, + } + }), + results, + } +} + +export {list, listForExperiment, get} +export type {Run, RunOutput, RunOutputResult} diff --git a/website/src/scenarios.ts b/website/src/scenarios.ts index d6ac3d2c..61de85e5 100644 --- a/website/src/scenarios.ts +++ b/website/src/scenarios.ts @@ -1,17 +1,15 @@ import fs from 'node:fs/promises' import path from 'node:path' -import type {ResolvedScenario} from '@primer/agent-eval/scenarios' +import type {Scenario as AgentEvalScenario} from '@primer/agent-eval/scenario' -const {listScenarios, findScenario} = await import( +const {listScenarios, getScenario} = await import( /* turbopackIgnore: true */ - '@primer/agent-eval/scenarios' + '@primer/agent-eval/scenario' ) const SCENARIOS_DIR = path.resolve(process.cwd(), '..', 'scenarios') -export type ScenarioSummary = Pick & { - id: string -} +export type ScenarioSummary = Pick export type Scenario = ScenarioSummary & { test: string @@ -22,28 +20,23 @@ export async function list(): Promise> { directory: SCENARIOS_DIR, }) - return scenarios - .filter(scenario => !scenario.id.startsWith('000-')) - .map(scenario => { - return { - id: scenario.id, - prompt: scenario.config.prompt, - } - }) + return scenarios.map(scenario => { + return { + id: scenario.id, + prompt: scenario.prompt, + } + }) } export async function get(id: string): Promise { - const scenario = await findScenario(id, { + const scenario = await getScenario({ directory: SCENARIOS_DIR, + id, }) - if (!scenario) { - throw new Error(`Scenario "${id}" was not found in: ${SCENARIOS_DIR}`) - } - return { id: scenario.id, - prompt: scenario.config.prompt, + prompt: scenario.prompt, test: await fs.readFile(scenario.testPath, 'utf8'), } }