test: add benchmark scenarios

.changeset/bright-plans-shard.md

@@ -0,0 +1,5 @@
+---
+'@primer/agent-eval': minor
+---
+
+Add durable plan creation, replay, deterministic sharding, and result merging to the CLI and public package API.

.github/workflows/benchmark.yml

@@ -25,8 +25,47 @@ permissions:
   contents: read
 
 jobs:
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      run-date: ${{ steps.run-date.outputs.value }}
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: build project
+        run: pnpm run build
+      - name: set run date
+        id: run-date
+        run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT"
+      - name: create benchmark plan
+        env:
+          RUN_DATE: ${{ steps.run-date.outputs.value }}
+        run: script/run-benchmark.sh design-system plan
+      - name: upload benchmark plan
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: benchmark-plan-${{ github.run_id }}
+          path: results
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 9
+
   run:
+    needs: plan
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        shard: [1, 2, 3, 4]
     steps:
       - name: checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -41,21 +80,66 @@ jobs:
         run: pnpm install --frozen-lockfile
       - name: build project
         run: pnpm run build
-      - name: run benchmark
+      - name: download benchmark plan
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: benchmark-plan-${{ github.run_id }}
+      - name: run benchmark shard
         env:
           CONCURRENCY: ${{ inputs.concurrency }}
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
-        run: script/run-benchmark.sh design-system
-      - name: prepare benchmark artifact
+          RUN_DATE: ${{ needs.plan.outputs.run-date }}
+          SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}
+        run: script/run-benchmark.sh design-system shard
+      - name: prepare benchmark shard artifact
         if: ${{ always() }}
         run: |
           mkdir -p workflow-artifact
           if [[ -d results ]]; then
             mv results workflow-artifact/results
           fi
-      - name: upload benchmark results
+      - name: upload benchmark shard
         if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: benchmark-shard-${{ github.run_id }}-${{ matrix.shard }}
+          path: workflow-artifact
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 9
+
+  merge:
+    needs: [plan, run]
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: build project
+        run: pnpm run build
+      - name: download benchmark shards
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: benchmark-shard-${{ github.run_id }}-*
+          merge-multiple: true
+      - name: merge benchmark shards
+        env:
+          RUN_DATE: ${{ needs.plan.outputs.run-date }}
+        run: script/run-benchmark.sh design-system merge
+      - name: prepare benchmark artifact
+        run: |
+          mkdir -p workflow-artifact
+          mv results workflow-artifact/results
+      - name: upload benchmark results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: benchmark-${{ github.run_id }}

.github/workflows/experiment.yml

@@ -27,37 +27,121 @@ permissions:
   contents: read
 
 jobs:
-  run:
+  plan:
     runs-on: ubuntu-latest
+    outputs:
+      run-date: ${{ steps.run-date.outputs.value }}
     steps:
       - name: checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: set up pnpm
-        uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
-          cache: true
-          require-lockfile: true
+          node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: build project
+        run: pnpm run build
+      - name: set run date
+        id: run-date
+        run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT"
+      - name: create experiment plan
+        env:
+          RUN_DATE: ${{ steps.run-date.outputs.value }}
+        run: script/run-experiment.sh "${{ inputs.experiment }}" plan
+      - name: upload experiment plan
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: experiment-plan-${{ github.run_id }}
+          path: results
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 9
+
+  run:
+    needs: plan
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        shard: [1, 2, 3, 4]
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
       - name: set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version-file: '.nvmrc'
+          cache: 'pnpm'
+      - name: install dependencies
+        run: pnpm install --frozen-lockfile
       - name: build project
         run: pnpm run build
-      - name: run experiment
+      - name: download experiment plan
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: experiment-plan-${{ github.run_id }}
+      - name: run experiment shard
         env:
-          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           CONCURRENCY: ${{ inputs.concurrency }}
+          COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
-        run: script/run-experiment.sh "${{ inputs.experiment }}"
-      - name: prepare experiment artifact
+          RUN_DATE: ${{ needs.plan.outputs.run-date }}
+          SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}
+        run: script/run-experiment.sh "${{ inputs.experiment }}" shard
+      - name: prepare experiment shard artifact
         if: ${{ always() }}
         run: |
           mkdir -p workflow-artifact
           if [[ -d results ]]; then
             mv results workflow-artifact/results
           fi
-      - name: upload experiment results
+      - name: upload experiment shard
         if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: experiment-shard-${{ github.run_id }}-${{ matrix.shard }}
+          path: workflow-artifact
+          if-no-files-found: error
+          retention-days: 1
+          compression-level: 9
+
+  merge:
+    needs: [plan, run]
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: set up pnpm
+        uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
+        with:
+          cache: true
+          require-lockfile: true
+      - name: set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: '.nvmrc'
+      - name: build project
+        run: pnpm run build
+      - name: download experiment shards
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: experiment-shard-${{ github.run_id }}-*
+          merge-multiple: true
+      - name: merge experiment shards
+        env:
+          RUN_DATE: ${{ needs.plan.outputs.run-date }}
+        run: script/run-experiment.sh "${{ inputs.experiment }}" merge
+      - name: prepare experiment artifact
+        run: |
+          mkdir -p workflow-artifact
+          mv results workflow-artifact/results
+      - name: upload experiment results
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: experiment-${{ github.run_id }}

packages/agent-eval/README.md

@@ -164,6 +164,44 @@ directory to preserve those references. `--output-dir` creates `output.json`
 and `artifacts/` within the selected directory. When using `--output`, artifacts
 are written to an `artifacts/` directory beside the selected file.
 
+### Plans and sharding
+
+Create a durable, randomized trial plan before running an experiment or
+benchmark:
+
+```sh
+agent-eval --experiment example --plan plan.json
+```
+
+Plan creation does not require a Copilot token. The plan stores the ordered
+trial IDs and references needed to reload the experiment or benchmark. Keep the
+same experiment, benchmark, and scenario configuration available when running
+the plan.
+
+Run deterministic shards from the shared plan, writing a distinct output file
+for each shard:
+
+```sh
+COPILOT_GITHUB_TOKEN=... agent-eval \
+  --from-plan plan.json \
+  --shard 1/4 \
+  --output-dir run
+```
+
+After all shards finish, merge the `output-*.json` files into one portable
+result:
+
+```sh
+agent-eval --merge-results --output-dir run
+```
+
+The merged `output.json` must stay in the same directory as the shard manifests
+so their per-trial file references remain portable.
+
+`--plan` and `--from-plan` default to `plan.json` when their path is omitted.
+With `--output-dir`, `--shard 1/4` writes `output-1.json`. `--shard` is only
+valid with `--from-plan`. Shard merging does not require a Copilot token.
+
 ## Scenario config authoring
 
 Use `defineConfig` from `@primer/agent-eval/scenario` in each

packages/agent-eval/package.json

@@ -18,6 +18,10 @@
       "types": "./dist/experiment.d.ts",
       "default": "./dist/experiment.js"
     },
+    "./plan": {
+      "types": "./dist/plan.d.ts",
+      "default": "./dist/plan.js"
+    },
     "./scenario": {
       "types": "./dist/scenario.d.ts",
       "default": "./dist/scenario.js"

packages/agent-eval/rolldown.config.ts

@@ -13,6 +13,7 @@ const config = defineConfig({
     cli: 'src/cli.ts',
     experiment: 'src/experiment.ts',
     index: 'src/index.ts',
+    plan: 'src/plan.ts',
     scenario: 'src/scenario.ts',
   },
   platform: 'node',