ci: align benchmark and experiment runners

.github/workflows/benchmark.yml

@@ -46,7 +46,7 @@ jobs:
           CONCURRENCY: ${{ inputs.concurrency }}
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
-        run: script/run-benchmark.sh
+        run: script/run-benchmark.sh design-system
       - name: prepare benchmark artifact
         if: ${{ always() }}
         run: |

.github/workflows/experiment.yml

@@ -27,101 +27,40 @@ permissions:
   contents: read
 
 jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      run-date: ${{ steps.run.outputs.date }}
-    steps:
-      - name: Set run date
-        id: run
-        run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT"
-
   run:
-    needs: setup
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: true
-      matrix:
-        order: [1, 2, 3, 4]
     steps:
-      - name: Checkout repository
+      - name: checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - name: Set up pnpm
+      - name: set up pnpm
         uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
         with:
           cache: true
           require-lockfile: true
-      - name: Set up Node.js
+      - name: set up Node.js
         uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version-file: '.nvmrc'
-      - name: Build project
+      - name: build project
         run: pnpm run build
-      - name: Run experiment
+      - name: run experiment
         env:
           COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
           CONCURRENCY: ${{ inputs.concurrency }}
           DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }}
-          EXPERIMENT_NAME: ${{ inputs.experiment }}
-          RUN_DATE: ${{ needs.setup.outputs.run-date }}
-          SHARD: ${{ matrix.order }}/${{ strategy.job-total }}
-        run: |
-          set -o pipefail
-          run_directory="results/experiments/$EXPERIMENT_NAME/$RUN_DATE"
-          node packages/agent-eval/dist/cli.js \
-            --concurrency "$CONCURRENCY" \
-            --docker-image "$DOCKER_IMAGE" \
-            --experiment "$EXPERIMENT_NAME" \
-            --experiments experiments \
-            --scenarios scenarios \
-            --output "$run_directory/output-${{ matrix.order }}.json" \
-            --shard "$SHARD"
-      - name: Prepare experiment artifact
+        run: script/run-experiment.sh "${{ inputs.experiment }}"
+      - name: prepare experiment artifact
         if: ${{ always() }}
         run: |
           mkdir -p workflow-artifact
           if [[ -d results ]]; then
             mv results workflow-artifact/results
           fi
-      - name: Upload experiment results
-        if: ${{ always() }}
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: experiment-shard-${{ github.run_id }}-${{ matrix.order }}
-          path: workflow-artifact
-          if-no-files-found: error
-          retention-days: 1
-          compression-level: 9
-
-  merge:
-    needs: [setup, run]
-    runs-on: ubuntu-latest
-    steps:
-      - name: download experiment results
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          pattern: experiment-shard-${{ github.run_id }}-*
-          merge-multiple: true
-      - name: merge experiment results
-        run: |
-          cd "results/experiments/${{ inputs.experiment }}/${{ needs.setup.outputs.run-date }}"
-          jq --slurp '
-            {
-              experimentId: .[0].experimentId,
-              scenarios: (reduce .[].scenarios as $scenarios ({}; . * $scenarios)),
-              treatments: (reduce .[].treatments as $treatments ({}; . * $treatments)),
-              trials: (reduce .[].trials as $trials ({}; . * $trials))
-            }
-          ' output-*.json > output.json
-          rm output-*.json
-      - name: Prepare experiment artifact
-        run: |
-          mkdir -p workflow-artifact
-          mv results workflow-artifact/results
       - name: upload experiment results
+        if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: experiment
+          name: experiment-${{ github.run_id }}
           path: workflow-artifact
           if-no-files-found: error
           retention-days: 90

script/run-benchmark.sh

@@ -3,12 +3,17 @@
 set -euo pipefail
 
 repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-benchmark_name="${BENCHMARK_NAME:-design-system}"
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <benchmark-name>" >&2
+  exit 1
+fi
+benchmark_name="$1"
 run_date="${RUN_DATE:-$(date -u +%F)}"
 run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date"
 
 if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
-  echo "BENCHMARK_NAME must be a benchmark file name without its extension" >&2
+  echo "Benchmark name must be a file name without its extension" >&2
   exit 1
 fi
 

script/run-experiment.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <experiment-name>" >&2
+  exit 1
+fi
+
+experiment_name="$1"
+run_date="${RUN_DATE:-$(date -u +%F)}"
+run_directory="$repository_root/results/experiments/$experiment_name/$run_date"
+
+if [[ ! "$experiment_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+  echo "Experiment name must be a file name without its extension" >&2
+  exit 1
+fi
+
+if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
+  echo "RUN_DATE must use the YYYY-MM-DD format" >&2
+  exit 1
+fi
+
+node "$repository_root/packages/agent-eval/bin/agent-eval" \
+  --experiment "$experiment_name" \
+  --experiments "$repository_root/experiments" \
+  --concurrency "${CONCURRENCY:-1}" \
+  --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \
+  --output-dir "$run_directory" \
+  --scenarios "$repository_root/scenarios"

script/setup

@@ -9,15 +9,15 @@ corepack install
 # Install dependencies
 pnpm install --frozen-lockfile
 
-# Download the 10 most recent retained benchmark result bundles.
+# Download the 10 most recent retained benchmark and experiment result bundles.
 artifact_ids="$(
   gh api \
     --paginate \
     --slurp \
     "/repos/primer/agent-eval/actions/artifacts?per_page=100" |
     jq --raw-output '
       [.[].artifacts[]
-        | select(.expired == false and (.name | test("^benchmark-[0-9]+$")))
+        | select(.expired == false and (.name | test("^(benchmark-[0-9]+|experiment(-[0-9]+)?)$")))
       ]
       | sort_by(.created_at)
       | .[-10:][].id