diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4dc419e7..0c721635 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -46,7 +46,7 @@ jobs: CONCURRENCY: ${{ inputs.concurrency }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - run: script/run-benchmark.sh + run: script/run-benchmark.sh design-system - name: prepare benchmark artifact if: ${{ always() }} run: | diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index bb8ad426..95437416 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -27,101 +27,40 @@ permissions: contents: read jobs: - setup: - runs-on: ubuntu-latest - outputs: - run-date: ${{ steps.run.outputs.date }} - steps: - - name: Set run date - id: run - run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" - run: - needs: setup runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - order: [1, 2, 3, 4] steps: - - name: Checkout repository + - name: checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up pnpm + - name: set up pnpm uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 with: cache: true require-lockfile: true - - name: Set up Node.js + - name: set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version-file: '.nvmrc' - - name: Build project + - name: build project run: pnpm run build - - name: Run experiment + - name: run experiment env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} CONCURRENCY: ${{ inputs.concurrency }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - EXPERIMENT_NAME: ${{ inputs.experiment }} - RUN_DATE: ${{ needs.setup.outputs.run-date }} - SHARD: ${{ matrix.order }}/${{ strategy.job-total }} - run: | - set -o pipefail - run_directory="results/experiments/$EXPERIMENT_NAME/$RUN_DATE" - node packages/agent-eval/dist/cli.js \ - --concurrency "$CONCURRENCY" \ - --docker-image "$DOCKER_IMAGE" \ - --experiment "$EXPERIMENT_NAME" \ - --experiments experiments \ - --scenarios scenarios \ - --output "$run_directory/output-${{ matrix.order }}.json" \ - --shard "$SHARD" - - name: Prepare experiment artifact + run: script/run-experiment.sh "${{ inputs.experiment }}" + - name: prepare experiment artifact if: ${{ always() }} run: | mkdir -p workflow-artifact if [[ -d results ]]; then mv results workflow-artifact/results fi - - name: Upload experiment results - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: experiment-shard-${{ github.run_id }}-${{ matrix.order }} - path: workflow-artifact - if-no-files-found: error - retention-days: 1 - compression-level: 9 - - merge: - needs: [setup, run] - runs-on: ubuntu-latest - steps: - - name: download experiment results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: experiment-shard-${{ github.run_id }}-* - merge-multiple: true - - name: merge experiment results - run: | - cd "results/experiments/${{ inputs.experiment }}/${{ needs.setup.outputs.run-date }}" - jq --slurp ' - { - experimentId: .[0].experimentId, - scenarios: (reduce .[].scenarios as $scenarios ({}; . * $scenarios)), - treatments: (reduce .[].treatments as $treatments ({}; . * $treatments)), - trials: (reduce .[].trials as $trials ({}; . * $trials)) - } - ' output-*.json > output.json - rm output-*.json - - name: Prepare experiment artifact - run: | - mkdir -p workflow-artifact - mv results workflow-artifact/results - name: upload experiment results + if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: experiment + name: experiment-${{ github.run_id }} path: workflow-artifact if-no-files-found: error retention-days: 90 diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh index 4f6b0da1..f0cae005 100755 --- a/script/run-benchmark.sh +++ b/script/run-benchmark.sh @@ -3,12 +3,17 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -benchmark_name="${BENCHMARK_NAME:-design-system}" + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi +benchmark_name="$1" run_date="${RUN_DATE:-$(date -u +%F)}" run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date" if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then - echo "BENCHMARK_NAME must be a benchmark file name without its extension" >&2 + echo "Benchmark name must be a file name without its extension" >&2 exit 1 fi diff --git a/script/run-experiment.sh b/script/run-experiment.sh new file mode 100755 index 00000000..e0123f3c --- /dev/null +++ b/script/run-experiment.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +experiment_name="$1" +run_date="${RUN_DATE:-$(date -u +%F)}" +run_directory="$repository_root/results/experiments/$experiment_name/$run_date" + +if [[ ! "$experiment_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "Experiment name must be a file name without its extension" >&2 + exit 1 +fi + +if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "RUN_DATE must use the YYYY-MM-DD format" >&2 + exit 1 +fi + +node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --experiment "$experiment_name" \ + --experiments "$repository_root/experiments" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" diff --git a/script/setup b/script/setup index 2305a4fd..fe3104ae 100755 --- a/script/setup +++ b/script/setup @@ -9,7 +9,7 @@ corepack install # Install dependencies pnpm install --frozen-lockfile -# Download the 10 most recent retained benchmark result bundles. +# Download the 10 most recent retained benchmark and experiment result bundles. artifact_ids="$( gh api \ --paginate \ @@ -17,7 +17,7 @@ artifact_ids="$( "/repos/primer/agent-eval/actions/artifacts?per_page=100" | jq --raw-output ' [.[].artifacts[] - | select(.expired == false and (.name | test("^benchmark-[0-9]+$"))) + | select(.expired == false and (.name | test("^(benchmark-[0-9]+|experiment(-[0-9]+)?)$"))) ] | sort_by(.created_at) | .[-10:][].id