feat: update project to support new experiment and benchmark structure

.changeset/bright-benchmarks-run.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add benchmark configuration, discovery, execution, and output APIs through `@primer/agent-eval/benchmark` and explicitly named package-root exports. The CLI can now select and run benchmarks from a benchmarks directory.

.changeset/calm-experiments-unify.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Move experiment APIs to `@primer/agent-eval/experiment`, replacing `findExperiment`, package-root `run`, and package-root `defineConfig` with `getExperiment`, `runExperiment`, and `defineExperimentConfig`. Remove the legacy experiment loading helpers in favor of `getExperiment` and `listExperiments` with explicit source directories.

.changeset/fresh-experiment-configs.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add `ExperimentConfigSchema` and update experiment configuration to use model variants, the new treatment API, and the new sandbox interface. Scenarios continue to support IDs and `{path, name?}` entries through the exported `ExperimentScenarioConfig` and `InlineScenarioConfig` types.

.changeset/gentle-scenarios-unify.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Move scenario APIs to `@primer/agent-eval/scenario`, replacing `defineScenario`, `findScenario`, and `ResolvedScenario` with `defineConfig`, `getScenario`, and `Scenario`. Add scenario schemas and remove the legacy `@primer/agent-eval/scenarios` entry point and `loadScenarioDirectory`.

.changeset/happy-output-maps.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Replace `createAgentEvalOutput`, `parseAgentEvalOutput`, `AgentEvalOutput`, and `AgentEvalOutputResult` with the experiment `output` and `deserialize` helpers, `ExperimentOutput`, and `TrialResult`. Experiment output now uses keyed maps, model variants, per-session agent metrics, and directory-oriented artifact fields.

.changeset/kind-trials-compare.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add package-root trial and treatment schemas, types, execution helpers, and comparison helpers, replacing `TreatmentResult` with `TrialResult`. Trials continue to run optional Playwright browser tests and combine them with standard scenario test results.

.changeset/lucky-sandboxes-open.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add `@primer/agent-eval/sandbox` for sandbox runtimes, configuration, constants, and the plugin and MCP types previously exported from the experiment entry point. Replace the concrete `Sandbox` class with the `Sandbox` interface plus `SystemSandbox` and `VirtualSandbox`.

.changeset/neat-models-vary.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Replace `ExperimentModelConfig`, `ModelInfo`, and `resolveModelConfigs` with model variant configuration, schemas, and expansion helpers. Low-level model helpers and types are no longer exported from the package root or experiment entry point.

.changeset/plain-scenarios-discover.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Update scenario discovery to support explicit hosts, skip template directories, and recognize `browser.test.ts`. Inline scenario paths remain supported without package discovery requirements and continue to recognize `scenario.browser.test.ts`.

.changeset/portable-results-bundle.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add `--output-dir` for portable experiment and benchmark bundles with artifact and walkthrough paths relative to `output.json`. Remove `--artifacts` and derive the artifact directory from `--output-dir` or the directory containing `--output`.

.changeset/quick-copilot-metrics.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Accept sub-agent `user.message` events without `agentMode` and preserve their routing fields. Collect output token counts from `model.message` events while retaining compatibility with older Copilot output.

.changeset/steady-trial-results.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Exclude the temporary `agent-browser` walkthrough skill from downloaded trial artifacts.

.changeset/tender-benchmarks-compare.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': patch
+---
+
+Compare benchmark test success rates instead of passed-test totals and report equal metrics as a 0% change.

.changeset/tidy-cli-entrypoint.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Remove the `@primer/agent-eval/cli` package entry point. Use the `agent-eval` executable for CLI usage and the package root or API entry points for programmatic usage.

.github/workflows/benchmark.yml

@@ -0,0 +1,65 @@
+name: benchmark
+on:
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+    inputs:
+      concurrency:
+        description: Number of trials to run in parallel
+        required: false
+        default: '1'
+        type: string
+      docker-image:
+        description: >-
+          Docker container image to use for running trials.
+          Must be a Debian-based Node image with apt-get and a node user (e.g. node:26.5.0-slim).
+        required: false
+        default: 'node:26.5.0-slim'
+        type: string
+
+concurrency:
+  group: weekly-design-system-benchmark
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: build project
+        run: pnpm run build
+      - name: run benchmark
+        env:
+          CONCURRENCY: ${{ inputs.concurrency }}
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
+          DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
+        run: script/run-benchmark.sh
+      - name: prepare benchmark artifact
+        if: ${{ always() }}
+        run: |
+          mkdir -p workflow-artifact
+          if [[ -d results ]]; then
+            mv results workflow-artifact/results
+          fi
+      - name: upload benchmark results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: benchmark-${{ github.run_id }}
+          path: workflow-artifact
+          if-no-files-found: error
+          retention-days: 90
+          compression-level: 9

.github/workflows/deploy.yml

@@ -5,7 +5,7 @@ on:
       - main
   workflow_run:
     workflows:
-      - baseline
+      - benchmark
     types:
       - completed
     branches:
@@ -31,13 +31,13 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: ${{ github.event.repository.default_branch }}
-      - name: download baseline results
+      - name: download evaluation results
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh api --paginate \
             "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[] | select(.expired == false and (.name | test("^baseline-[0-9]+$"))) | [.created_at, .id] | @tsv' |
+            --jq '.artifacts[] | select(.expired == false and (.name | test("^benchmark-[0-9]+$"))) | [.created_at, .id] | @tsv' |
             sort |
             cut -f2 |
             while read -r artifact_id; do

.github/workflows/experiment.yml

@@ -27,7 +27,17 @@ permissions:
   contents: read
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      run-date: ${{ steps.run.outputs.date }}
+    steps:
+      - name: Set run date
+        id: run
+        run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT"
+
   run:
+    needs: setup
     runs-on: ubuntu-latest
     strategy:
       fail-fast: true
@@ -53,33 +63,38 @@ jobs:
           CONCURRENCY: ${{ inputs.concurrency }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
           EXPERIMENT_NAME: ${{ inputs.experiment }}
-          OUTPUT_FILE: output-${{ matrix.order }}.json
+          RUN_DATE: ${{ needs.setup.outputs.run-date }}
           SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
         run: |
           set -o pipefail
+          run_directory="results/experiments/$EXPERIMENT_NAME/$RUN_DATE"
           node packages/agent-eval/dist/cli.js \
             --concurrency "$CONCURRENCY" \
             --docker-image "$DOCKER_IMAGE" \
             --experiment "$EXPERIMENT_NAME" \
             --experiments experiments \
             --scenarios scenarios \
-            --artifacts artifacts \
-            --output "$OUTPUT_FILE" \
+            --output "$run_directory/output-${{ matrix.order }}.json" \
             --shard "$SHARD"
+      - name: Prepare experiment artifact
+        if: ${{ always() }}
+        run: |
+          mkdir -p workflow-artifact
+          if [[ -d results ]]; then
+            mv results workflow-artifact/results
+          fi
       - name: Upload experiment results
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: experiment-shard-${{ github.run_id }}-${{ matrix.order }}
-          path: |
-            output-${{ matrix.order }}.json
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 1
           compression-level: 9
 
   merge:
-    needs: run
+    needs: [setup, run]
     runs-on: ubuntu-latest
     steps:
       - name: download experiment results
@@ -89,27 +104,25 @@ jobs:
           merge-multiple: true
       - name: merge experiment results
         run: |
+          cd "results/experiments/${{ inputs.experiment }}/${{ needs.setup.outputs.run-date }}"
           jq --slurp '
-            . as $outputs
-            | ([$outputs[].treatments[]] | unique_by(.config.name)) as $treatments
-            | $outputs[0]
-            | .treatments = $treatments
-            | .results = [
-                $outputs[] as $output
-                | $output.results[]
-                | . as $result
-                | ($output.treatments[] | select(.id == $result.treatmentId).config.name) as $name
-                | .treatmentId = ($treatments[] | select(.config.name == $name).id)
-              ]
+            {
+              experimentId: .[0].experimentId,
+              scenarios: (reduce .[].scenarios as $scenarios ({}; . * $scenarios)),
+              treatments: (reduce .[].treatments as $treatments ({}; . * $treatments)),
+              trials: (reduce .[].trials as $trials ({}; . * $trials))
+            }
           ' output-*.json > output.json
           rm output-*.json
+      - name: Prepare experiment artifact
+        run: |
+          mkdir -p workflow-artifact
+          mv results workflow-artifact/results
       - name: upload experiment results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: experiment
-          path: |
-            output.json
-            artifacts
+          path: workflow-artifact
           if-no-files-found: error
           retention-days: 90
           compression-level: 9

.github/workflows/report-workflow-failures.yml

@@ -3,7 +3,7 @@ name: Report workflow failures
 on:
   workflow_run:
     workflows:
-      - baseline
+      - benchmark
       - deploy
       - update-pnpm
       - update-sandbox-tools

.gitignore

@@ -31,3 +31,4 @@ out
 # Generated files
 artifacts/
 results/
+output.json