diff --git a/.changeset/bright-plans-shard.md b/.changeset/bright-plans-shard.md new file mode 100644 index 00000000..463ec723 --- /dev/null +++ b/.changeset/bright-plans-shard.md @@ -0,0 +1,5 @@ +--- +'@primer/agent-eval': minor +--- + +Add durable plan creation, replay, deterministic sharding, and result merging to the CLI and public package API. diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0c721635..8f5ce6bd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -25,8 +25,47 @@ permissions: contents: read jobs: + plan: + runs-on: ubuntu-latest + outputs: + run-date: ${{ steps.run-date.outputs.value }} + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: set run date + id: run-date + run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT" + - name: create benchmark plan + env: + RUN_DATE: ${{ steps.run-date.outputs.value }} + run: script/run-benchmark.sh design-system plan + - name: upload benchmark plan + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-plan-${{ github.run_id }} + path: results + if-no-files-found: error + retention-days: 1 + compression-level: 9 + run: + needs: plan runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + shard: [1, 2, 3, 4] steps: - name: checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -41,21 +80,66 @@ jobs: run: pnpm install --frozen-lockfile - name: build project run: pnpm run build - - name: run benchmark + - name: download benchmark plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: benchmark-plan-${{ github.run_id }} + - name: run benchmark shard env: CONCURRENCY: ${{ inputs.concurrency }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - run: script/run-benchmark.sh design-system - - name: prepare benchmark artifact + RUN_DATE: ${{ needs.plan.outputs.run-date }} + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + run: script/run-benchmark.sh design-system shard + - name: prepare benchmark shard artifact if: ${{ always() }} run: | mkdir -p workflow-artifact if [[ -d results ]]; then mv results workflow-artifact/results fi - - name: upload benchmark results + - name: upload benchmark shard if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-shard-${{ github.run_id }}-${{ matrix.shard }} + path: workflow-artifact + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + merge: + needs: [plan, run] + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: download benchmark shards + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: benchmark-shard-${{ github.run_id }}-* + merge-multiple: true + - name: merge benchmark shards + env: + RUN_DATE: ${{ needs.plan.outputs.run-date }} + run: script/run-benchmark.sh design-system merge + - name: prepare benchmark artifact + run: | + mkdir -p workflow-artifact + mv results workflow-artifact/results + - name: upload benchmark results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: benchmark-${{ github.run_id }} diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 95437416..b54961f4 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -27,37 +27,121 @@ permissions: contents: read jobs: - run: + plan: runs-on: ubuntu-latest + outputs: + run-date: ${{ steps.run-date.outputs.value }} steps: - name: checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: set up pnpm - uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: - cache: true - require-lockfile: true + node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile + - name: build project + run: pnpm run build + - name: set run date + id: run-date + run: echo "value=$(date -u +%F)" >> "$GITHUB_OUTPUT" + - name: create experiment plan + env: + RUN_DATE: ${{ steps.run-date.outputs.value }} + run: script/run-experiment.sh "${{ inputs.experiment }}" plan + - name: upload experiment plan + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: experiment-plan-${{ github.run_id }} + path: results + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + run: + needs: plan + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + shard: [1, 2, 3, 4] + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - name: set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version-file: '.nvmrc' + cache: 'pnpm' + - name: install dependencies + run: pnpm install --frozen-lockfile - name: build project run: pnpm run build - - name: run experiment + - name: download experiment plan + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: experiment-plan-${{ github.run_id }} + - name: run experiment shard env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} CONCURRENCY: ${{ inputs.concurrency }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} DOCKER_IMAGE: ${{ inputs.docker-image || 'node:26.5.0-slim' }} - run: script/run-experiment.sh "${{ inputs.experiment }}" - - name: prepare experiment artifact + RUN_DATE: ${{ needs.plan.outputs.run-date }} + SHARD: ${{ matrix.shard }}/${{ strategy.job-total }} + run: script/run-experiment.sh "${{ inputs.experiment }}" shard + - name: prepare experiment shard artifact if: ${{ always() }} run: | mkdir -p workflow-artifact if [[ -d results ]]; then mv results workflow-artifact/results fi - - name: upload experiment results + - name: upload experiment shard if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: experiment-shard-${{ github.run_id }}-${{ matrix.shard }} + path: workflow-artifact + if-no-files-found: error + retention-days: 1 + compression-level: 9 + + merge: + needs: [plan, run] + runs-on: ubuntu-latest + steps: + - name: checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: set up pnpm + uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0 + with: + cache: true + require-lockfile: true + - name: set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: '.nvmrc' + - name: build project + run: pnpm run build + - name: download experiment shards + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: experiment-shard-${{ github.run_id }}-* + merge-multiple: true + - name: merge experiment shards + env: + RUN_DATE: ${{ needs.plan.outputs.run-date }} + run: script/run-experiment.sh "${{ inputs.experiment }}" merge + - name: prepare experiment artifact + run: | + mkdir -p workflow-artifact + mv results workflow-artifact/results + - name: upload experiment results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: experiment-${{ github.run_id }} diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 528ab803..a160b54f 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -164,6 +164,44 @@ directory to preserve those references. `--output-dir` creates `output.json` and `artifacts/` within the selected directory. When using `--output`, artifacts are written to an `artifacts/` directory beside the selected file. +### Plans and sharding + +Create a durable, randomized trial plan before running an experiment or +benchmark: + +```sh +agent-eval --experiment example --plan plan.json +``` + +Plan creation does not require a Copilot token. The plan stores the ordered +trial IDs and references needed to reload the experiment or benchmark. Keep the +same experiment, benchmark, and scenario configuration available when running +the plan. + +Run deterministic shards from the shared plan, writing a distinct output file +for each shard: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval \ + --from-plan plan.json \ + --shard 1/4 \ + --output-dir run +``` + +After all shards finish, merge the `output-*.json` files into one portable +result: + +```sh +agent-eval --merge-results --output-dir run +``` + +The merged `output.json` must stay in the same directory as the shard manifests +so their per-trial file references remain portable. + +`--plan` and `--from-plan` default to `plan.json` when their path is omitted. +With `--output-dir`, `--shard 1/4` writes `output-1.json`. `--shard` is only +valid with `--from-plan`. Shard merging does not require a Copilot token. + ## Scenario config authoring Use `defineConfig` from `@primer/agent-eval/scenario` in each diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index 0344e8a5..d06e56d7 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -18,6 +18,10 @@ "types": "./dist/experiment.d.ts", "default": "./dist/experiment.js" }, + "./plan": { + "types": "./dist/plan.d.ts", + "default": "./dist/plan.js" + }, "./scenario": { "types": "./dist/scenario.d.ts", "default": "./dist/scenario.js" diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index 7e1d288b..dfae1a90 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -13,6 +13,7 @@ const config = defineConfig({ cli: 'src/cli.ts', experiment: 'src/experiment.ts', index: 'src/index.ts', + plan: 'src/plan.ts', scenario: 'src/scenario.ts', }, platform: 'node', diff --git a/packages/agent-eval/src/benchmark.test.ts b/packages/agent-eval/src/benchmark.test.ts index 314a75e5..dd6f5a5c 100644 --- a/packages/agent-eval/src/benchmark.test.ts +++ b/packages/agent-eval/src/benchmark.test.ts @@ -1,16 +1,20 @@ import {afterEach, expect, test, vi} from 'vitest' import { + createPlan, defineConfig, getBenchmark, listBenchmarks, + merge, output, read, + resolvePlan, run, write, + type Benchmark, type BenchmarkTrialResult, } from './benchmark' import {VirtualHost} from './host' -import {run as runPlan} from './plan' +import {deserialize as deserializePlan, isBenchmarkPlan, run as runPlan} from './plan' import {defineConfig as defineScenarioConfig} from './scenario' vi.mock('./plan', async importOriginal => { @@ -440,4 +444,105 @@ test('writes and reads benchmark capability metadata', async () => { 'utf-8', ) await expect(read('/bundle/embedded-output.json', {host})).rejects.toThrow() + + const secondOutput = output('test-benchmark', [ + { + ...trialResult, + trial: { + ...trialResult.trial, + id: 'trial-two', + }, + }, + ]) + const merged = merge([benchmarkOutput, secondOutput]) + + expect(merged.capabilities.get('Test capability')).toEqual({ + name: 'Test capability', + scenarioIds: ['001-scenario'], + }) + expect([...merged.trials.keys()]).toEqual(['trial', 'trial-two']) + expect(() => { + merge([benchmarkOutput, benchmarkOutput]) + }).toThrow('Cannot merge duplicate trial id: trial') +}) + +test('resolves durable benchmark plans with capability metadata and setup functions', () => { + const setup = async () => { + return undefined + } + const benchmark: Benchmark = { + id: 'test-benchmark', + filepath: '/benchmarks/test-benchmark.ts', + name: 'Test benchmark', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + setup, + capabilities: [ + { + name: 'Test capability', + scenarios: [ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }, + ], + }, + ], + } + + const plan = createPlan(benchmark) + const resolved = resolvePlan(benchmark, plan) + const benchmarkTrial = resolved.plan.trials.find(trial => trial.treatment.name === 'Benchmark') + + expect(resolved.plan.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(benchmarkTrial?.treatment.setup).toBe(setup) + expect(resolved.trialCapabilities.get(benchmarkTrial?.id ?? '')).toBe(benchmark.capabilities[0]) +}) + +test('rejects benchmark plans with references missing from the current config', () => { + const benchmark: Benchmark = { + id: 'test-benchmark', + filepath: '/benchmarks/test-benchmark.ts', + name: 'Test benchmark', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + capabilities: [], + } + const plan = deserializePlan({ + version: 1, + source: { + kind: 'benchmark', + id: benchmark.id, + }, + trials: [ + { + id: 'trial', + scenarioId: '001-scenario', + treatmentId: 'Benchmark', + model: benchmark.models[0], + capabilityId: 'Missing capability', + }, + ], + }) + + if (!isBenchmarkPlan(plan)) { + throw new Error('Expected benchmark plan') + } + + expect(() => { + resolvePlan(benchmark, plan) + }).toThrow('Plan trial "trial" references missing benchmark capability: Missing capability') }) diff --git a/packages/agent-eval/src/benchmark.ts b/packages/agent-eval/src/benchmark.ts index a8287a3b..901d85c4 100644 --- a/packages/agent-eval/src/benchmark.ts +++ b/packages/agent-eval/src/benchmark.ts @@ -5,7 +5,13 @@ import type {EnvironmentConfig} from './environment' import {DefaultHost, type Host} from './host' import {logger} from './logger' import {getModelVariants, ModelVariantConfigSchema, ModelVariantSchema, type ModelVariant} from './model' -import {create as createPlan, run as runPlan} from './plan' +import { + create as createDurablePlan, + run as runPlan, + type BenchmarkPlan, + type BenchmarkPlanTrialReference, + type RuntimePlan, +} from './plan' import {getScenario, ScenarioSchema, type Scenario} from './scenario' import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' import { @@ -174,48 +180,153 @@ type BenchmarkTrialResult = TrialResult & { type BenchmarkRunResult = Array +function createPlan(benchmark: Benchmark): BenchmarkPlan { + const capabilityIds = new Set() + + for (const capability of benchmark.capabilities) { + if (capabilityIds.has(capability.name)) { + throw new Error(`Benchmark "${benchmark.id}" contains duplicate capability id: ${capability.name}`) + } + + capabilityIds.add(capability.name) + } + + const trials: Array = benchmark.models.flatMap(model => { + return benchmark.capabilities.flatMap(capability => { + return capability.scenarios.flatMap(scenario => { + return [ControlTreatment.name, 'Benchmark'].map(treatmentId => { + return { + id: randomUUID(), + scenarioId: scenario.id, + treatmentId, + model, + capabilityId: capability.name, + } + }) + }) + }) + }) + + return createDurablePlan({ + source: { + kind: 'benchmark', + id: benchmark.id, + }, + trials, + }) +} + +function resolvePlan( + benchmark: Benchmark, + plan: BenchmarkPlan, +): { + plan: RuntimePlan + trialCapabilities: Map +} { + if (plan.source.kind !== 'benchmark') { + throw new Error(`Expected a benchmark plan, received: ${plan.source.kind}`) + } + + if (plan.source.id !== benchmark.id) { + throw new Error(`Plan references benchmark "${plan.source.id}", but loaded benchmark "${benchmark.id}"`) + } + + const capabilities = new Map() + for (const capability of benchmark.capabilities) { + if (capabilities.has(capability.name)) { + throw new Error(`Benchmark "${benchmark.id}" contains duplicate capability id: ${capability.name}`) + } + + capabilities.set(capability.name, capability) + } + const trialCapabilities = new Map() + const trials: Array = plan.trials.map(reference => { + const capability = capabilities.get(reference.capabilityId) + if (!capability) { + throw new Error(`Plan trial "${reference.id}" references missing benchmark capability: ${reference.capabilityId}`) + } + + const scenario = capability.scenarios.find(candidate => candidate.id === reference.scenarioId) + if (!scenario) { + throw new Error( + `Plan trial "${reference.id}" references scenario "${reference.scenarioId}" outside capability "${reference.capabilityId}"`, + ) + } + + const model = benchmark.models.find(candidate => { + return candidate.name === reference.model.name && candidate.reasoningEffort === reference.model.reasoningEffort + }) + if (!model) { + throw new Error( + `Plan trial "${reference.id}" references missing model variant: ${reference.model.name}/${reference.model.reasoningEffort}`, + ) + } + + let treatment + if (reference.treatmentId === ControlTreatment.name) { + treatment = ControlTreatment + } else if (reference.treatmentId === 'Benchmark') { + treatment = createBenchmarkTreatment(benchmark, capability) + } else { + throw new Error(`Plan trial "${reference.id}" references missing benchmark treatment: ${reference.treatmentId}`) + } + + trialCapabilities.set(reference.id, capability) + return { + id: reference.id, + scenario, + treatment, + model, + } + }) + + return { + plan: { + trials, + }, + trialCapabilities, + } +} + async function run({ env, host = DefaultHost, id, + plan, }: { env: EnvironmentConfig host?: Host - id: string + id?: string + plan?: BenchmarkPlan }): Promise { + if (id && plan) { + throw new Error('Benchmark run accepts either an id or a plan, not both') + } + + if (!id && !plan) { + throw new Error('Benchmark run requires an id or a plan') + } + + const benchmarkId = plan?.source.id ?? id + if (!benchmarkId) { + throw new Error('Benchmark run requires an id or a plan') + } + const benchmark = await getBenchmark({ host, benchmarksDirectory: env.benchmarksDirectory, scenariosDirectory: env.scenariosDirectory, - id, + id: benchmarkId, }) - const trialCapabilities = new Map() - const trials: Array = benchmark.models.flatMap(model => { - return benchmark.capabilities.flatMap(capability => { - const benchmarkTreatment = createBenchmarkTreatment(benchmark, capability) - return capability.scenarios.flatMap(scenario => { - return [ControlTreatment, benchmarkTreatment].map(treatment => { - const trial = { - id: randomUUID(), - scenario, - treatment, - model, - } - trialCapabilities.set(trial.id, capability) - return trial - }) - }) - }) - }) - const plan = await createPlan(trials) + const resolved = resolvePlan(benchmark, plan ?? createPlan(benchmark)) const results = await runPlan({ env, host, - plan, + plan: resolved.plan, }) return results.map(result => { - const capability = trialCapabilities.get(result.trial.id) + const capability = resolved.trialCapabilities.get(result.trial.id) if (!capability) { throw new Error(`Capability was not found for trial: ${result.trial.id}`) } @@ -228,13 +339,13 @@ async function run({ } function createBenchmarkTreatment(benchmark: Benchmark, capability: Capability): Treatment { - const setup = - benchmark.setup || capability.setup - ? async ({sandbox}: Parameters[0]) => { - await benchmark.setup?.({sandbox}) - await capability.setup?.({sandbox}) - } - : undefined + let setup = benchmark.setup ?? capability.setup + if (benchmark.setup && capability.setup) { + setup = async ({sandbox}: Parameters[0]) => { + await benchmark.setup?.({sandbox}) + await capability.setup?.({sandbox}) + } + } return { name: 'Benchmark', @@ -283,6 +394,8 @@ const BenchmarkOutputFileSchema = z.object({ trials: z.record(z.string(), z.string()), }) +type BenchmarkOutputFile = z.infer + type BenchmarkOutput = { benchmarkId: string capabilities: Map> @@ -386,11 +499,78 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } -export {BenchmarkConfigSchema, defineConfig, getBenchmark, listBenchmarks, output, read, run, write} +function parseOutputFile(input: unknown): BenchmarkOutputFile { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + return BenchmarkOutputFileSchema.parse(parsed, {reportInput: true}) +} + +function merge(outputs: Array): BenchmarkOutput { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one benchmark output is required to merge shards') + } + + const result: BenchmarkOutput = { + benchmarkId: first.benchmarkId, + capabilities: new Map(first.capabilities), + scenarios: new Map(first.scenarios), + treatments: new Map(first.treatments), + trials: new Map(first.trials), + } + + for (const shardOutput of remaining) { + if (shardOutput.benchmarkId !== result.benchmarkId) { + throw new Error( + `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${shardOutput.benchmarkId}"`, + ) + } + + mergeMetadataMap(result.capabilities, shardOutput.capabilities, 'capability') + mergeMetadataMap(result.scenarios, shardOutput.scenarios, 'scenario') + mergeMetadataMap(result.treatments, shardOutput.treatments, 'treatment') + + for (const [trialId, trial] of shardOutput.trials) { + if (result.trials.has(trialId)) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } + + result.trials.set(trialId, trial) + } + } + + return result +} + +function mergeMetadataMap(target: Map, source: Map, type: string): void { + for (const [id, value] of source) { + const existing = target.get(id) + if (target.has(id) && JSON.stringify(existing) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target.set(id, value) + } +} + +export { + BenchmarkConfigSchema, + createPlan, + defineConfig, + getBenchmark, + listBenchmarks, + merge, + output, + parseOutputFile, + read, + resolvePlan, + run, + write, +} export type { BenchmarkConfig, Benchmark, BenchmarkOutput, + BenchmarkOutputFile, BenchmarkOutputOptions, BenchmarkRunResult, BenchmarkTrialResult, diff --git a/packages/agent-eval/src/cli-options.test.ts b/packages/agent-eval/src/cli-options.test.ts new file mode 100644 index 00000000..63d04f5c --- /dev/null +++ b/packages/agent-eval/src/cli-options.test.ts @@ -0,0 +1,83 @@ +import {describe, expect, test} from 'vitest' +import {getCliMode, normalizeOptionalPathArguments} from './cli-options' + +describe('normalizeOptionalPathArguments', () => { + test('adds defaults for bare optional path flags', () => { + expect(normalizeOptionalPathArguments(['--benchmark', 'test', '--plan'])).toEqual([ + '--benchmark', + 'test', + '--plan=plan.json', + ]) + expect(normalizeOptionalPathArguments(['--from-plan', '--shard', '2/3'])).toEqual([ + '--from-plan=plan.json', + '--shard', + '2/3', + ]) + }) + + test('preserves explicit optional paths', () => { + expect(normalizeOptionalPathArguments(['--plan', 'plans/test.json'])).toEqual(['--plan', 'plans/test.json']) + }) +}) + +describe('getCliMode', () => { + test('creates plan and from-plan modes', () => { + expect( + getCliMode({ + benchmark: 'test', + plan: 'plan.json', + }), + ).toEqual({ + kind: 'create-plan', + sourceKind: 'benchmark', + sourceId: 'test', + path: 'plan.json', + }) + expect( + getCliMode({ + 'from-plan': 'plan.json', + shard: '2/3', + }), + ).toEqual({ + kind: 'from-plan', + path: 'plan.json', + shard: '2/3', + }) + }) + + test('validates incompatible modes', () => { + expect(() => { + getCliMode({ + benchmark: 'benchmark', + experiment: 'experiment', + }) + }).toThrow('--benchmark and --experiment cannot be combined') + + expect(() => { + getCliMode({ + benchmark: 'benchmark', + 'from-plan': 'plan.json', + }) + }).toThrow('--from-plan cannot be combined') + + expect(() => { + getCliMode({ + benchmark: 'benchmark', + shard: '1/2', + }) + }).toThrow('--shard is only valid with --from-plan') + + expect(() => { + getCliMode({ + experiment: 'experiment', + 'merge-results': true, + }) + }).toThrow('--merge-results cannot be combined') + + expect(() => { + getCliMode({ + plan: 'plan.json', + }) + }).toThrow('--plan requires --benchmark or --experiment') + }) +}) diff --git a/packages/agent-eval/src/cli-options.ts b/packages/agent-eval/src/cli-options.ts new file mode 100644 index 00000000..3fbe8ad9 --- /dev/null +++ b/packages/agent-eval/src/cli-options.ts @@ -0,0 +1,141 @@ +const DEFAULT_PLAN_PATH = 'plan.json' + +const optionalPathDefaults = new Map([ + ['--plan', DEFAULT_PLAN_PATH], + ['--from-plan', DEFAULT_PLAN_PATH], +]) + +type CliModeOptions = { + benchmark?: string + experiment?: string + plan?: string + 'from-plan'?: string + 'merge-results'?: boolean + shard?: string +} + +type CliMode = + | { + kind: 'none' + } + | { + kind: 'benchmark' + id: string + } + | { + kind: 'experiment' + id: string + } + | { + kind: 'create-plan' + sourceKind: 'benchmark' | 'experiment' + sourceId: string + path: string + } + | { + kind: 'from-plan' + path: string + shard?: string + } + | { + kind: 'merge-results' + } + +function normalizeOptionalPathArguments(args: Array): Array { + const normalized: Array = [] + + for (let index = 0; index < args.length; index++) { + const argument = args[index] + const defaultValue = optionalPathDefaults.get(argument) + if (defaultValue === undefined) { + normalized.push(argument) + continue + } + + const next = args[index + 1] + if (next === undefined || next.startsWith('-')) { + normalized.push(`${argument}=${defaultValue}`) + continue + } + + normalized.push(argument, next) + index += 1 + } + + return normalized +} + +function getCliMode(options: CliModeOptions): CliMode { + if (options.benchmark && options.experiment) { + throw new Error('--benchmark and --experiment cannot be combined') + } + + if (options['from-plan']) { + if (options.benchmark || options.experiment || options.plan !== undefined || options['merge-results']) { + throw new Error('--from-plan cannot be combined with --benchmark, --experiment, --plan, or --merge-results') + } + + return { + kind: 'from-plan', + path: options['from-plan'], + shard: options.shard, + } + } + + if (options['merge-results']) { + if (options.benchmark || options.experiment || options.plan !== undefined) { + throw new Error('--merge-results cannot be combined with --benchmark, --experiment, or --plan') + } + + if (options.shard) { + throw new Error('--shard is only valid with --from-plan') + } + + return { + kind: 'merge-results', + } + } + + if (options.shard) { + throw new Error('--shard is only valid with --from-plan') + } + + if (options.plan !== undefined) { + if (!options.benchmark && !options.experiment) { + throw new Error('--plan requires --benchmark or --experiment') + } + + const sourceId = options.benchmark ?? options.experiment + if (!sourceId) { + throw new Error('--plan requires --benchmark or --experiment') + } + + return { + kind: 'create-plan', + sourceKind: options.benchmark ? 'benchmark' : 'experiment', + sourceId, + path: options.plan, + } + } + + if (options.benchmark) { + return { + kind: 'benchmark', + id: options.benchmark, + } + } + + if (options.experiment) { + return { + kind: 'experiment', + id: options.experiment, + } + } + + return { + kind: 'none', + } +} + +export {DEFAULT_PLAN_PATH, getCliMode, normalizeOptionalPathArguments} +export type {CliMode, CliModeOptions} diff --git a/packages/agent-eval/src/cli.test.ts b/packages/agent-eval/src/cli.test.ts index 7f42fdba..d919fddd 100644 --- a/packages/agent-eval/src/cli.test.ts +++ b/packages/agent-eval/src/cli.test.ts @@ -1,3 +1,6 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' import {afterEach, describe, expect, test, vi} from 'vitest' const originalArgv = process.argv @@ -25,10 +28,12 @@ describe('cli', () => { await expect(import('./cli')).rejects.toThrow('process.exit') expect(log).toHaveBeenCalledWith(expect.stringContaining('Usage: agent-eval [options]')) expect(log).toHaveBeenCalledWith(expect.stringContaining('--output-dir ')) + expect(log).toHaveBeenCalledWith(expect.stringContaining('--from-plan [path]')) + expect(log).toHaveBeenCalledWith(expect.stringContaining('--merge-results')) }) - test('requires a Copilot token before running', async () => { - process.argv = ['node', 'agent-eval'] + test('requires a Copilot token when running', async () => { + process.argv = ['node', 'agent-eval', '--benchmark', 'test'] delete process.env.COPILOT_GITHUB_TOKEN await expect(import('./cli')).rejects.toThrow( @@ -38,7 +43,7 @@ describe('cli', () => { test('displays help when no benchmark or experiment is selected', async () => { process.argv = ['node', 'agent-eval'] - process.env.COPILOT_GITHUB_TOKEN = 'token' + delete process.env.COPILOT_GITHUB_TOKEN const log = vi.spyOn(console, 'log').mockImplementation(() => {}) await import('./cli') @@ -52,4 +57,24 @@ describe('cli', () => { await expect(import('./cli')).rejects.toThrow('--output-dir cannot be combined with --output') }) + + test('does not require a Copilot token when creating a plan', async () => { + const benchmarksDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-benchmarks-')) + + try { + process.argv = ['node', 'agent-eval', '--benchmark', 'test', '--benchmarks', benchmarksDirectory, '--plan'] + delete process.env.COPILOT_GITHUB_TOKEN + + await expect(import('./cli')).rejects.toThrow('Benchmark "test" was not found') + } finally { + await fs.rm(benchmarksDirectory, {recursive: true}) + } + }) + + test('does not require a Copilot token when merging shard outputs', async () => { + process.argv = ['node', 'agent-eval', '--merge-results', '--output-dir', '/missing-agent-eval-shards'] + delete process.env.COPILOT_GITHUB_TOKEN + + await expect(import('./cli')).rejects.toThrow('ENOENT') + }) }) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index 95b2b072..82e27128 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -3,25 +3,39 @@ import fs from 'node:fs/promises' import path from 'node:path' import {parseArgs} from 'node:util' +import {getCliMode, normalizeOptionalPathArguments} from './cli-options' import {getEnvironmentConfig} from './environment' import { + createPlan as createBenchmarkPlan, getBenchmark, run as runBenchmark, output as getBenchmarkOutput, write as writeBenchmarkOutput, } from './benchmark' import { + createPlan as createExperimentPlan, getExperiment, run as runExperiment, output as getExperimentOutput, write as writeExperimentOutput, } from './experiment' import {logger} from './logger' +import { + deserialize as deserializePlan, + isBenchmarkPlan, + mergeResults, + select as selectPlan, + serialize as serializePlan, + type BenchmarkPlan, + type ExperimentPlan, + type Plan, +} from './plan' import {formatBenchmarkResults, formatExperimentResults} from './report' import {parseShard} from './shard' import {compare as compareTrial} from './trial' const {values} = parseArgs({ + args: normalizeOptionalPathArguments(process.argv.slice(2)), options: { benchmark: { type: 'string', @@ -72,9 +86,21 @@ const {values} = parseArgs({ type: 'string', description: 'The directory containing scenario directories', }, + plan: { + type: 'string', + description: 'Create a durable plan without running it', + }, + 'from-plan': { + type: 'string', + description: 'Run trials from a durable plan', + }, + 'merge-results': { + type: 'boolean', + description: 'Merge output-*.json shard outputs from a directory', + }, shard: { type: 'string', - description: 'The experiment shard to run, formatted as order/total', + description: 'The durable plan shard to run, formatted as order/total', }, }, }) @@ -94,8 +120,11 @@ Options: --log-level The log level to use (default: info) --output The target file in which results are written (default: output.json) --output-dir The directory containing output.json and its artifacts + --plan [path] Create a durable plan without running it (default: plan.json) + --from-plan [path] Run trials from a durable plan (default: plan.json) + --merge-results Merge output-*.json files in --output-dir --scenarios The directory containing scenario directories (default: ./scenarios) - --shard The experiment shard to run + --shard Select a deterministic shard from --from-plan `) } @@ -110,35 +139,81 @@ if (values['log-level']) { const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN const GITHUB_STEP_SUMMARY = process.env.GITHUB_STEP_SUMMARY - -if (!COPILOT_GITHUB_TOKEN) { - throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval') -} +const mode = getCliMode(values) +const shard = mode.kind === 'from-plan' && mode.shard ? parseShard(mode.shard) : undefined const env = getEnvironmentConfig({ benchmarksDirectory: values.benchmarks, concurrency: values.concurrency, - copilotToken: COPILOT_GITHUB_TOKEN, + copilotToken: COPILOT_GITHUB_TOKEN ?? '', dockerImage: values['docker-image']?.trim(), experimentsDirectory: values.experiments, outputDirectory: values['output-dir'], outputPath: values.output, scenariosDirectory: values.scenarios, + shard, }) logger.debug('Environment configuration: %o', env) -if (values.benchmark) { - logger.info('Running benchmark: %s', values.benchmark) +if (mode.kind === 'create-plan') { + const planPath = path.resolve(mode.path) + let plan: Plan + + if (mode.sourceKind === 'benchmark') { + const benchmark = await getBenchmark({ + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id: mode.sourceId, + }) + plan = createBenchmarkPlan(benchmark) + } else { + const experiment = await getExperiment({ + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id: mode.sourceId, + }) + plan = createExperimentPlan(experiment) + } + + await ensureParentDirectory(planPath) + logger.info('Writing plan to: %s', planPath) + await fs.writeFile(planPath, serializePlan(plan), 'utf-8') +} else if (mode.kind === 'merge-results') { + const directory = path.dirname(env.outputPath) + const entries = await fs.readdir(directory, { + withFileTypes: true, + }) + const filenames = entries + .filter(entry => { + return entry.isFile() && /^output-.*\.json$/.test(entry.name) + }) + .map(entry => { + return entry.name + }) + .toSorted() + const inputs = filenames.map(filename => { + return path.join(directory, filename) + }) + const merged = await mergeResults(inputs, { + targetDirectory: path.dirname(env.outputPath), + }) + + await ensureParentDirectory(env.outputPath) + logger.info('Writing merged %s output to: %s', merged.kind, env.outputPath) + await fs.writeFile(env.outputPath, JSON.stringify(merged.output), 'utf-8') +} else if (mode.kind === 'benchmark') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + logger.info('Running benchmark: %s', mode.id) const benchmark = await getBenchmark({ benchmarksDirectory: env.benchmarksDirectory, scenariosDirectory: env.scenariosDirectory, - id: values.benchmark, + id: mode.id, }) const result = await runBenchmark({ env, - id: values.benchmark, + id: mode.id, }) const sorted = result.toSorted(compareTrial) @@ -156,18 +231,18 @@ if (values.benchmark) { if (GITHUB_STEP_SUMMARY) { await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } -} else if (values.experiment) { - logger.info('Running experiment: %s', values.experiment) +} else if (mode.kind === 'experiment') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + logger.info('Running experiment: %s', mode.id) const experiment = await getExperiment({ experimentsDirectory: env.experimentsDirectory, scenariosDirectory: env.scenariosDirectory, - id: values.experiment, + id: mode.id, }) const result = await runExperiment({ env, - id: values.experiment, - shard: values.shard ? parseShard(values.shard) : undefined, + id: mode.id, }) const sorted = result.toSorted(compareTrial) @@ -185,6 +260,93 @@ if (values.benchmark) { if (GITHUB_STEP_SUMMARY) { await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) } +} else if (mode.kind === 'from-plan') { + requireCopilotToken(COPILOT_GITHUB_TOKEN) + const planPath = path.resolve(mode.path) + const durablePlan = deserializePlan(await fs.readFile(planPath, 'utf-8')) + const plan = shard ? selectDurablePlan(durablePlan, shard) : durablePlan + + if (isBenchmarkPlan(plan)) { + await runBenchmarkFromPlan(plan) + } else { + await runExperimentFromPlan(plan) + } } else { displayHelp() } + +function requireCopilotToken(token: string | undefined): asserts token is string { + if (!token) { + throw new Error('COPILOT_GITHUB_TOKEN environment variable is required to run agent-eval') + } +} + +function selectDurablePlan(plan: Plan, selectedShard: ReturnType): Plan { + if (isBenchmarkPlan(plan)) { + return selectPlan(plan, selectedShard) + } + + return selectPlan(plan, selectedShard) +} + +async function ensureParentDirectory(filepath: string): Promise { + await fs.mkdir(path.dirname(filepath), {recursive: true}) +} + +async function runBenchmarkFromPlan(plan: BenchmarkPlan): Promise { + logger.info('Running benchmark plan: %s', plan.source.id) + + const benchmark = await getBenchmark({ + benchmarksDirectory: env.benchmarksDirectory, + scenariosDirectory: env.scenariosDirectory, + id: plan.source.id, + }) + const result = await runBenchmark({ + env, + plan, + }) + const sorted = result.toSorted(compareTrial) + + await ensureParentDirectory(env.outputPath) + await writeBenchmarkOutput( + env.outputPath, + getBenchmarkOutput(benchmark.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }), + ) + + const resultSummaries = formatBenchmarkResults(benchmark, sorted) + console.log(resultSummaries) + + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Benchmark results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) + } +} + +async function runExperimentFromPlan(plan: ExperimentPlan): Promise { + logger.info('Running experiment plan: %s', plan.source.id) + + const experiment = await getExperiment({ + experimentsDirectory: env.experimentsDirectory, + scenariosDirectory: env.scenariosDirectory, + id: plan.source.id, + }) + const result = await runExperiment({ + env, + plan, + }) + const sorted = result.toSorted(compareTrial) + const output = getExperimentOutput(experiment.id, sorted, { + baseDirectory: path.dirname(env.outputPath), + }) + + await ensureParentDirectory(env.outputPath) + await writeExperimentOutput(env.outputPath, output) + + const resultSummaries = formatExperimentResults(experiment.name, sorted) + console.log(resultSummaries) + + if (GITHUB_STEP_SUMMARY) { + await fs.appendFile(GITHUB_STEP_SUMMARY, `## Experiment results\n\n\`\`\`\n${resultSummaries}\n\`\`\`\n`) + } +} diff --git a/packages/agent-eval/src/environment.test.ts b/packages/agent-eval/src/environment.test.ts index b6ded377..c9253c02 100644 --- a/packages/agent-eval/src/environment.test.ts +++ b/packages/agent-eval/src/environment.test.ts @@ -62,6 +62,28 @@ describe('getEnvironmentConfig', () => { }) }) + test('derives the output filename from the shard within an output directory', () => { + expect( + getEnvironmentConfig({ + copilotToken: 'token', + outputDirectory: './results/run', + shard: { + order: 2, + total: 4, + }, + }), + ).toEqual({ + artifactsDirectory: path.resolve('results/run/artifacts'), + benchmarksDirectory: path.resolve('benchmarks'), + concurrency: 1, + copilotToken: 'token', + dockerImage: DEFAULT_DOCKER_IMAGE, + experimentsDirectory: path.resolve('experiments'), + outputPath: path.resolve('results/run/output-2.json'), + scenariosDirectory: path.resolve('scenarios'), + }) + }) + test('rejects output directory combinations with explicit output paths', () => { expect(() => { getEnvironmentConfig({ diff --git a/packages/agent-eval/src/environment.ts b/packages/agent-eval/src/environment.ts index 8ce00d19..f24868db 100644 --- a/packages/agent-eval/src/environment.ts +++ b/packages/agent-eval/src/environment.ts @@ -1,5 +1,6 @@ import path from 'node:path' import {DEFAULT_DOCKER_IMAGE} from './sandbox' +import type {Shard} from './shard' type EnvironmentConfig = { artifactsDirectory: string @@ -21,6 +22,7 @@ type EnvironmentOptions = { outputDirectory?: string outputPath?: string scenariosDirectory?: string + shard?: Shard } function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig { @@ -37,7 +39,7 @@ function getEnvironmentConfig(options: EnvironmentOptions): EnvironmentConfig { : 1 const experimentsDirectory = path.resolve(options.experimentsDirectory ?? 'experiments') const outputPath = outputDirectory - ? path.join(outputDirectory, 'output.json') + ? path.join(outputDirectory, options.shard ? `output-${options.shard.order}.json` : 'output.json') : path.resolve(options.outputPath ?? 'output.json') const artifactsDirectory = path.join(path.dirname(outputPath), 'artifacts') const scenariosDirectory = path.resolve(options.scenariosDirectory ?? 'scenarios') diff --git a/packages/agent-eval/src/experiment.test.ts b/packages/agent-eval/src/experiment.test.ts index b71c42c6..0d349161 100644 --- a/packages/agent-eval/src/experiment.test.ts +++ b/packages/agent-eval/src/experiment.test.ts @@ -1,8 +1,20 @@ import path from 'node:path' import {expect, test} from 'vitest' -import {defineConfig, getExperiment, listExperiments, output, read, write} from './experiment' +import { + createPlan, + defineConfig, + getExperiment, + listExperiments, + merge, + output, + read, + resolvePlan, + write, + type Experiment, +} from './experiment' import {VirtualHost} from './host' import type {TrialResult} from './trial' +import {deserialize as deserializePlan, isBenchmarkPlan} from './plan' const config = defineConfig({ name: 'Test experiment', @@ -404,4 +416,166 @@ test('creates portable artifact paths relative to the output directory', async ( await expect(write('/bundle/output.json', portableOutput, {host})).rejects.toThrow( 'must not resolve outside the output directory', ) + + const secondOutput = output('baseline', [ + { + ...trialResult, + trial: { + ...trialResult.trial, + id: 'trial-two', + }, + }, + ]) + const merged = merge([portableOutput, secondOutput]) + + expect([...merged.trials.keys()]).toEqual(['trial', 'trial-two']) + expect(() => { + merge([portableOutput, portableOutput]) + }).toThrow('Cannot merge duplicate trial id: trial') + expect(() => { + merge([portableOutput, output('different', [])]) + }).toThrow('Cannot merge experiment outputs for different sources') +}) + +test('resolves durable experiment plans with experiment and treatment setup functions', () => { + const experimentSetup = async () => { + return undefined + } + const treatmentSetup = async () => { + return undefined + } + const experiment: Experiment = { + id: 'test-experiment', + filepath: '/experiments/test-experiment.ts', + name: 'Test experiment', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + scenarios: [ + { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + }, + ], + setup: experimentSetup, + treatments: [ + { + name: 'Treatment', + setup: treatmentSetup, + }, + ], + } + + const plan = createPlan(experiment) + const resolved = resolvePlan(experiment, plan) + const treatmentTrial = resolved.trials.find(trial => trial.treatment.name === 'Treatment') + + expect(resolved.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(treatmentTrial?.setup).toBe(experimentSetup) + expect(treatmentTrial?.treatment.setup).toBe(treatmentSetup) +}) + +test('rejects experiment plans with references missing from the current config', () => { + const scenario = { + id: '001-scenario', + directory: '/scenarios/001-scenario', + prompt: 'Complete the task', + tags: [], + testPath: '/scenarios/001-scenario/scenario.test.ts', + } + const experiment: Experiment = { + id: 'test-experiment', + filepath: '/experiments/test-experiment.ts', + name: 'Test experiment', + description: 'Tests durable plans', + models: [ + { + name: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }, + ], + scenarios: [scenario], + treatments: [], + } + const missingScenarioPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: 'missing-scenario', + treatmentId: 'Control', + model: experiment.models[0], + }, + ], + }) + + if (isBenchmarkPlan(missingScenarioPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingScenarioPlan) + }).toThrow('Plan trial "trial" references missing scenario: missing-scenario') + + const missingTreatmentPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: scenario.id, + treatmentId: 'Missing treatment', + model: experiment.models[0], + }, + ], + }) + + if (isBenchmarkPlan(missingTreatmentPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingTreatmentPlan) + }).toThrow('Plan trial "trial" references missing treatment: Missing treatment') + + const missingModelPlan = deserializePlan({ + version: 1, + source: { + kind: 'experiment', + id: experiment.id, + }, + trials: [ + { + id: 'trial', + scenarioId: scenario.id, + treatmentId: 'Control', + model: { + name: 'gpt-5.6-sol', + reasoningEffort: 'high', + }, + }, + ], + }) + + if (isBenchmarkPlan(missingModelPlan)) { + throw new Error('Expected experiment plan') + } + + expect(() => { + resolvePlan(experiment, missingModelPlan) + }).toThrow('Plan trial "trial" references missing model variant: gpt-5.6-sol/high') }) diff --git a/packages/agent-eval/src/experiment.ts b/packages/agent-eval/src/experiment.ts index 5d36a84a..26b2a865 100644 --- a/packages/agent-eval/src/experiment.ts +++ b/packages/agent-eval/src/experiment.ts @@ -11,7 +11,13 @@ import { } from './model' import {DefaultHost, type Host} from './host' import {logger} from './logger' -import {create as createPlan, run as runPlan} from './plan' +import { + create as createDurablePlan, + run as runPlan, + type ExperimentPlan, + type ExperimentPlanTrialReference, + type RuntimePlan, +} from './plan' import {getScenario, loadScenario, ScenarioSchema, type Scenario} from './scenario' import {selectShard, type Shard} from './shard' import {ControlTreatment, TreatmentSchema, TreatmentSetupSchema, type Treatment, type TreatmentSetup} from './treatment' @@ -23,7 +29,6 @@ import { WalkthroughSchema, writeTrialFiles, type ResultFileOptions, - type Trial, type TrialResult, } from './trial' import {TestResultsSchema} from './vitest' @@ -179,50 +184,132 @@ async function getExperiment({ type ExperimentRunResult = Array +function createPlan(experiment: Experiment): ExperimentPlan { + const treatmentIds = new Set([ControlTreatment.name]) + + for (const treatment of experiment.treatments) { + if (treatmentIds.has(treatment.name)) { + throw new Error(`Experiment "${experiment.id}" contains duplicate treatment id: ${treatment.name}`) + } + + treatmentIds.add(treatment.name) + } + + const trials: Array = experiment.models.flatMap(model => { + return experiment.scenarios.flatMap(scenario => { + return [ControlTreatment, ...experiment.treatments].map(treatment => { + return { + id: randomUUID(), + scenarioId: scenario.id, + treatmentId: treatment.name, + model, + } + }) + }) + }) + + return createDurablePlan({ + source: { + kind: 'experiment', + id: experiment.id, + }, + trials, + }) +} + +function resolvePlan(experiment: Experiment, plan: ExperimentPlan): RuntimePlan { + if (plan.source.kind !== 'experiment') { + throw new Error(`Expected an experiment plan, received: ${plan.source.kind}`) + } + + if (plan.source.id !== experiment.id) { + throw new Error(`Plan references experiment "${plan.source.id}", but loaded experiment "${experiment.id}"`) + } + + const treatments = new Map([[ControlTreatment.name, ControlTreatment]]) + for (const treatment of experiment.treatments) { + if (treatments.has(treatment.name)) { + throw new Error(`Experiment "${experiment.id}" contains duplicate treatment id: ${treatment.name}`) + } + + treatments.set(treatment.name, treatment) + } + + return { + trials: plan.trials.map(reference => { + const scenario = experiment.scenarios.find(candidate => candidate.id === reference.scenarioId) + if (!scenario) { + throw new Error(`Plan trial "${reference.id}" references missing scenario: ${reference.scenarioId}`) + } + + const treatment = treatments.get(reference.treatmentId) + if (!treatment) { + throw new Error(`Plan trial "${reference.id}" references missing treatment: ${reference.treatmentId}`) + } + + const model = experiment.models.find(candidate => { + return candidate.name === reference.model.name && candidate.reasoningEffort === reference.model.reasoningEffort + }) + if (!model) { + throw new Error( + `Plan trial "${reference.id}" references missing model variant: ${reference.model.name}/${reference.model.reasoningEffort}`, + ) + } + + return { + id: reference.id, + scenario, + treatment, + model, + setup: experiment.setup, + } + }), + } +} + async function run({ env, host = DefaultHost, id, + plan, shard, }: { env: EnvironmentConfig host?: Host - id: string + id?: string + plan?: ExperimentPlan shard?: Shard }): Promise { + if (id && plan) { + throw new Error('Experiment run accepts either an id or a plan, not both') + } + + if (!id && !plan) { + throw new Error('Experiment run requires an id or a plan') + } + + const experimentId = plan?.source.id ?? id + if (!experimentId) { + throw new Error('Experiment run requires an id or a plan') + } + const experiment = await getExperiment({ host, experimentsDirectory: env.experimentsDirectory, scenariosDirectory: env.scenariosDirectory, - id, + id: experimentId, }) - const trials: Array = experiment.models.flatMap(model => { - return experiment.scenarios.flatMap(scenario => { - return [ - { - id: randomUUID(), - scenario, - treatment: ControlTreatment, - model, - setup: experiment.setup, - }, - ...experiment.treatments.map(treatment => { - return { - id: randomUUID(), - scenario, - treatment, - model, - setup: experiment.setup, - } - }), - ] - }) - }) - const plan = await createPlan(shard ? selectShard(trials, shard) : trials) + const durablePlan = plan ?? createPlan(experiment) + const selectedPlan = shard + ? { + ...durablePlan, + trials: selectShard(durablePlan.trials, shard), + } + : durablePlan const results = await runPlan({ env, host, - plan, + plan: resolvePlan(experiment, selectedPlan), }) return results @@ -271,6 +358,8 @@ const ExperimentOutputFileSchema = z.object({ trials: z.record(z.string(), z.string()), }) +type ExperimentOutputFile = z.infer + function output( experimentId: string, trialResults: ExperimentRunResult, @@ -351,11 +440,76 @@ async function read(filepath: string, options: ResultFileOptions = {}): Promise< } } -export {ExperimentConfigSchema, defineConfig, getExperiment, listExperiments, output, read, run, write} +function parseOutputFile(input: unknown): ExperimentOutputFile { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + return ExperimentOutputFileSchema.parse(parsed, {reportInput: true}) +} + +function merge(outputs: Array): ExperimentOutput { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one experiment output is required to merge shards') + } + + const result: ExperimentOutput = { + experimentId: first.experimentId, + scenarios: new Map(first.scenarios), + treatments: new Map(first.treatments), + trials: new Map(first.trials), + } + + for (const shardOutput of remaining) { + if (shardOutput.experimentId !== result.experimentId) { + throw new Error( + `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${shardOutput.experimentId}"`, + ) + } + + mergeMetadataMap(result.scenarios, shardOutput.scenarios, 'scenario') + mergeMetadataMap(result.treatments, shardOutput.treatments, 'treatment') + + for (const [trialId, trial] of shardOutput.trials) { + if (result.trials.has(trialId)) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } + + result.trials.set(trialId, trial) + } + } + + return result +} + +function mergeMetadataMap(target: Map, source: Map, type: string): void { + for (const [id, value] of source) { + const existing = target.get(id) + if (target.has(id) && JSON.stringify(existing) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target.set(id, value) + } +} + +export { + ExperimentConfigSchema, + createPlan, + defineConfig, + getExperiment, + listExperiments, + merge, + output, + parseOutputFile, + read, + resolvePlan, + run, + write, +} export type { ExperimentConfig, Experiment, ExperimentOutput, + ExperimentOutputFile, ExperimentOutputOptions, ExperimentScenarioConfig, InlineScenarioConfig, diff --git a/packages/agent-eval/src/index.test.ts b/packages/agent-eval/src/index.test.ts index c74c9077..24cc47ca 100644 --- a/packages/agent-eval/src/index.test.ts +++ b/packages/agent-eval/src/index.test.ts @@ -3,6 +3,7 @@ import { BenchmarkConfigSchema, ControlTreatment, ExperimentConfigSchema, + PlanSchema, ScenarioConfigSchema, TreatmentSchema, TrialResultSchema, @@ -10,6 +11,8 @@ import { defineBenchmarkConfig, defineExperimentConfig, defineScenarioConfig, + deserializePlan, + serializePlan, } from './index' test('exports the public configuration helpers and schemas', () => { @@ -22,4 +25,7 @@ test('exports the public configuration helpers and schemas', () => { expect(TreatmentSchema.parse(ControlTreatment)).toEqual(ControlTreatment) expect(TrialSchema).toBeDefined() expect(TrialResultSchema).toBeDefined() + expect(PlanSchema).toBeDefined() + expect(deserializePlan).toBeTypeOf('function') + expect(serializePlan).toBeTypeOf('function') }) diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts index 1b63f017..8f39084e 100644 --- a/packages/agent-eval/src/index.ts +++ b/packages/agent-eval/src/index.ts @@ -1,10 +1,13 @@ export { BenchmarkConfigSchema, + createPlan as createBenchmarkPlan, defineConfig as defineBenchmarkConfig, getBenchmark, listBenchmarks, + merge as mergeBenchmarkOutputs, output as getBenchmarkOutput, read as readBenchmarkOutput, + resolvePlan as resolveBenchmarkPlan, run as runBenchmark, write as writeBenchmarkOutput, } from './benchmark' @@ -20,11 +23,14 @@ export type { export { ExperimentConfigSchema, + createPlan as createExperimentPlan, defineConfig as defineExperimentConfig, getExperiment, listExperiments, + merge as mergeExperimentOutputs, output as getExperimentOutput, read as readExperimentOutput, + resolvePlan as resolveExperimentPlan, run as runExperiment, write as writeExperimentOutput, } from './experiment' @@ -38,3 +44,28 @@ export type {Treatment} from './treatment' export {TrialSchema, TrialResultSchema, run as runTrial, compare as compareTrial} from './trial' export type {Trial, TrialResult} from './trial' + +export { + BenchmarkPlanSchema, + ExperimentPlanSchema, + PLAN_VERSION, + PlanSchema, + create as createPlan, + deserialize as deserializePlan, + isBenchmarkPlan, + mergeResults as mergePlanResults, + select as selectPlan, + serialize as serializePlan, +} from './plan' +export type { + BenchmarkPlan, + BenchmarkPlanTrialReference, + CreatePlanInput, + ExperimentPlan, + ExperimentPlanTrialReference, + MergedResults, + MergeResultsOptions, + Plan, + PlanTrialReference, + RuntimePlan, +} from './plan' diff --git a/packages/agent-eval/src/plan.test.ts b/packages/agent-eval/src/plan.test.ts index f85de6f8..d7a8d07e 100644 --- a/packages/agent-eval/src/plan.test.ts +++ b/packages/agent-eval/src/plan.test.ts @@ -1,6 +1,8 @@ import {afterEach, describe, expect, test, vi} from 'vitest' +import {output as getBenchmarkOutput, write as writeBenchmarkOutput, type BenchmarkOutput} from './benchmark' +import {output as getExperimentOutput, write as writeExperimentOutput, type ExperimentOutput} from './experiment' import {VirtualHost} from './host' -import {create, run} from './plan' +import {create, deserialize, isBenchmarkPlan, mergeResults, run, select, serialize} from './plan' import {run as runTrial} from './trial' import type {Trial, TrialResult} from './trial' @@ -65,16 +67,101 @@ function createResult(trial: Trial): TrialResult { } } +function createReference(id: string) { + return { + id, + scenarioId: 'scenario', + treatmentId: 'Control', + model: { + name: 'gpt-5.6-sol' as const, + reasoningEffort: 'medium' as const, + }, + } +} + +async function writeBenchmarkShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeBenchmarkOutput(filepath, output, {host}) + return filepath + }), + ) +} + +async function writeExperimentShards(host: VirtualHost, outputs: Array): Promise> { + return Promise.all( + outputs.map(async (output, index) => { + const filepath = `/bundle/output-${index + 1}.json` + await writeExperimentOutput(filepath, output, {host}) + return filepath + }), + ) +} + describe('create', () => { test('randomizes trials without mutating the input', async () => { - const trials = [createTrial('one'), createTrial('two'), createTrial('three')] + const trials = [createReference('one'), createReference('two'), createReference('three')] vi.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(0) - const plan = await create(trials) + const plan = create({ + source: { + kind: 'experiment', + id: 'test', + }, + trials, + }) expect(plan.trials.map(trial => trial.id)).toEqual(['two', 'three', 'one']) expect(trials.map(trial => trial.id)).toEqual(['one', 'two', 'three']) }) + + test('serializes and deserializes the durable plan without changing order or ids', () => { + const plan = create({ + source: { + kind: 'experiment', + id: 'test', + }, + trials: [createReference('one'), createReference('two')], + }) + + const restored = deserialize(serialize(plan)) + + expect(restored).toEqual(plan) + expect(restored.trials.map(trial => trial.id)).toEqual(plan.trials.map(trial => trial.id)) + expect(JSON.parse(serialize(plan))).toEqual(plan) + expect(serialize(plan)).not.toContain('setup') + }) + + test('rejects invalid durable plans', () => { + expect(() => { + deserialize({ + version: 1, + source: { + kind: 'benchmark', + id: 'test', + }, + trials: [createReference('trial')], + }) + }).toThrow() + }) + + test('selects deterministic shards from durable plan order', () => { + const plan = deserialize({ + version: 1, + source: { + kind: 'experiment', + id: 'test', + }, + trials: ['one', 'two', 'three', 'four', 'five'].map(createReference), + }) + + if (isBenchmarkPlan(plan)) { + throw new Error('Expected experiment plan') + } + + expect(select(plan, {order: 2, total: 3}).trials.map(trial => trial.id)).toEqual(['two', 'five']) + }) }) describe('run', () => { @@ -166,3 +253,118 @@ describe('run', () => { expect(runTrial).toHaveBeenCalledTimes(4) }) }) + +describe('mergeResults', () => { + test('detects and merges benchmark shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('benchmark', []), + getBenchmarkOutput('benchmark', []), + ]) + const merged = await mergeResults(filepaths, {host}) + + expect(merged.kind).toBe('benchmark') + expect(merged.output).toEqual({ + benchmarkId: 'benchmark', + capabilities: {}, + scenarios: {}, + treatments: {}, + trials: {}, + }) + }) + + test('detects and merges experiment shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [ + getExperimentOutput('experiment', []), + getExperimentOutput('experiment', []), + ]) + const merged = await mergeResults(filepaths, {host}) + + expect(merged.kind).toBe('experiment') + expect(merged.output).toEqual({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: {}, + }) + }) + + test('rejects mixed output types', async () => { + const host = VirtualHost.create() + await writeBenchmarkOutput('/bundle/output-1.json', getBenchmarkOutput('benchmark', []), {host}) + await writeExperimentOutput('/bundle/output-2.json', getExperimentOutput('experiment', []), {host}) + + await expect( + mergeResults(['/bundle/output-1.json', '/bundle/output-2.json'], { + host, + }), + ).rejects.toThrow('Cannot merge benchmark and experiment shard outputs together') + }) + + test('requires shard outputs from one source id', async () => { + const host = VirtualHost.create() + const filepaths = await writeBenchmarkShards(host, [ + getBenchmarkOutput('first', []), + getBenchmarkOutput('second', []), + ]) + + await expect(mergeResults(filepaths, {host})).rejects.toThrow( + 'Cannot merge benchmark outputs for different sources', + ) + }) + + test('combines trial file references without reading the trial files', async () => { + const host = VirtualHost.create() + await host.fs.mkdir('/bundle', {recursive: true}) + await host.fs.writeFile( + '/bundle/output-1.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + first: 'artifacts/first/first.json', + }, + }), + 'utf-8', + ) + await host.fs.writeFile( + '/bundle/output-2.json', + JSON.stringify({ + experimentId: 'experiment', + scenarios: {}, + treatments: {}, + trials: { + second: 'artifacts/second/second.json', + }, + }), + 'utf-8', + ) + + const merged = await mergeResults(['/bundle/output-1.json', '/bundle/output-2.json'], { + host, + }) + + if (merged.kind !== 'experiment') { + throw new Error('Expected experiment output') + } + + expect(merged.output.trials).toEqual({ + first: 'artifacts/first/first.json', + second: 'artifacts/second/second.json', + }) + }) + + test('requires the merged output to stay beside the shard outputs', async () => { + const host = VirtualHost.create() + const filepaths = await writeExperimentShards(host, [getExperimentOutput('experiment', [])]) + + await expect( + mergeResults(filepaths, { + host, + targetDirectory: '/merged', + }), + ).rejects.toThrow('Shard outputs and the merged output must use the same directory') + }) +}) diff --git a/packages/agent-eval/src/plan.ts b/packages/agent-eval/src/plan.ts index 9d503cc8..9b75bef5 100644 --- a/packages/agent-eval/src/plan.ts +++ b/packages/agent-eval/src/plan.ts @@ -1,28 +1,253 @@ +import path from 'node:path' import Queue from 'p-queue' -import {run as runTrial} from './trial' -import type {Trial, TrialResult} from './trial' +import * as z from 'zod/mini' +import type {BenchmarkOutputFile} from './benchmark' import type {EnvironmentConfig} from './environment' +import type {ExperimentOutputFile} from './experiment' import {DefaultHost, type Host} from './host' import {logger} from './logger' +import {ModelVariantSchema} from './model' +import {selectShard, type Shard} from './shard' +import {run as runTrial} from './trial' +import type {Trial, TrialResult} from './trial' + +const PLAN_VERSION = 1 + +const BenchmarkPlanTrialReferenceSchema = z.object({ + id: z.string(), + scenarioId: z.string(), + treatmentId: z.string(), + model: ModelVariantSchema, + capabilityId: z.string(), +}) + +const ExperimentPlanTrialReferenceSchema = z.object({ + id: z.string(), + scenarioId: z.string(), + treatmentId: z.string(), + model: ModelVariantSchema, +}) + +const BenchmarkPlanSchema = z.object({ + version: z.literal(PLAN_VERSION), + source: z.object({ + kind: z.literal('benchmark'), + id: z.string(), + }), + trials: z.array(BenchmarkPlanTrialReferenceSchema), +}) + +const ExperimentPlanSchema = z.object({ + version: z.literal(PLAN_VERSION), + source: z.object({ + kind: z.literal('experiment'), + id: z.string(), + }), + trials: z.array(ExperimentPlanTrialReferenceSchema), +}) + +const PlanSchema = z.union([BenchmarkPlanSchema, ExperimentPlanSchema]) + +type BenchmarkPlanTrialReference = z.infer +type ExperimentPlanTrialReference = z.infer +type BenchmarkPlan = z.infer +type ExperimentPlan = z.infer -/** - * A plan is an ordered list of trials to be ran. - */ -type Plan = { +type Plan = BenchmarkPlan | ExperimentPlan +type PlanTrialReference = BenchmarkPlanTrialReference | ExperimentPlanTrialReference +type CreatePlanInput = Omit | Omit + +type RuntimePlan = { trials: Array } -// TODO: support plan with sharding -async function create(trials: Array): Promise { +type MergedResults = + | { + kind: 'benchmark' + output: BenchmarkOutputFile + } + | { + kind: 'experiment' + output: ExperimentOutputFile + } + +type MergeResultsOptions = { + host?: Host + targetDirectory?: string +} + +function create(input: Omit): BenchmarkPlan +function create(input: Omit): ExperimentPlan +function create(input: CreatePlanInput): Plan { + return input.source.kind === 'benchmark' + ? BenchmarkPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) + : ExperimentPlanSchema.parse({ + version: PLAN_VERSION, + source: input.source, + trials: randomize(input.trials), + }) +} + +function serialize(plan: Plan): string { + const parsed = PlanSchema.parse(plan) + return `${JSON.stringify(parsed, null, 2)}\n` +} + +function deserialize(input: unknown): Plan { + const parsed = typeof input === 'string' ? JSON.parse(input) : input + return PlanSchema.parse(parsed, {reportInput: true}) +} + +function select(plan: BenchmarkPlan, shard: Shard): BenchmarkPlan +function select(plan: ExperimentPlan, shard: Shard): ExperimentPlan +function select(plan: Plan, shard: Shard): Plan { + if (isBenchmarkPlan(plan)) { + return { + ...plan, + trials: selectShard(plan.trials, shard), + } + } + return { - trials: randomize(trials), + ...plan, + trials: selectShard(plan.trials, shard), + } +} + +function isBenchmarkPlan(plan: Plan): plan is BenchmarkPlan { + return plan.source.kind === 'benchmark' +} + +async function mergeResults(filepaths: Array, options: MergeResultsOptions = {}): Promise { + if (filepaths.length === 0) { + throw new Error('No shard outputs were found to merge') + } + + const targetDirectory = path.resolve(options.targetDirectory ?? path.dirname(filepaths[0])) + for (const filepath of filepaths) { + if (path.resolve(path.dirname(filepath)) !== targetDirectory) { + throw new Error('Shard outputs and the merged output must use the same directory') + } + } + + const host = options.host ?? DefaultHost + const manifests = await Promise.all( + filepaths.map(async filepath => { + return JSON.parse(await host.fs.readFile(filepath, 'utf-8')) as unknown + }), + ) + const kinds = manifests.map(getOutputKind) + const firstKind = kinds[0] + if ( + kinds.some(kind => { + return kind !== firstKind + }) + ) { + throw new Error('Cannot merge benchmark and experiment shard outputs together') + } + + if (firstKind === 'benchmark') { + const {parseOutputFile} = await import('./benchmark') + return { + kind: 'benchmark', + output: mergeBenchmarkOutputFiles(manifests.map(parseOutputFile)), + } + } + + const {parseOutputFile} = await import('./experiment') + return { + kind: 'experiment', + output: mergeExperimentOutputFiles(manifests.map(parseOutputFile)), + } +} + +function getOutputKind(input: unknown): 'benchmark' | 'experiment' { + if (typeof input !== 'object' || input === null) { + throw new Error('Shard output must be a JSON object') + } + + const hasBenchmarkId = 'benchmarkId' in input + const hasExperimentId = 'experimentId' in input + if (hasBenchmarkId === hasExperimentId) { + throw new Error('Shard output must contain exactly one of benchmarkId or experimentId') + } + + return hasBenchmarkId ? 'benchmark' : 'experiment' +} + +function mergeBenchmarkOutputFiles(outputs: Array): BenchmarkOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one benchmark output is required to merge shards') + } + + const result = structuredClone(first) + for (const output of remaining) { + if (output.benchmarkId !== result.benchmarkId) { + throw new Error( + `Cannot merge benchmark outputs for different sources: "${result.benchmarkId}" and "${output.benchmarkId}"`, + ) + } + + mergeMetadataRecord(result.capabilities, output.capabilities, 'capability') + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) + } + + return result +} + +function mergeExperimentOutputFiles(outputs: Array): ExperimentOutputFile { + const [first, ...remaining] = outputs + if (!first) { + throw new Error('At least one experiment output is required to merge shards') + } + + const result = structuredClone(first) + for (const output of remaining) { + if (output.experimentId !== result.experimentId) { + throw new Error( + `Cannot merge experiment outputs for different sources: "${result.experimentId}" and "${output.experimentId}"`, + ) + } + + mergeMetadataRecord(result.scenarios, output.scenarios, 'scenario') + mergeMetadataRecord(result.treatments, output.treatments, 'treatment') + mergeTrialReferences(result.trials, output.trials) + } + + return result +} + +function mergeMetadataRecord(target: Record, source: Record, type: string): void { + for (const [id, value] of Object.entries(source)) { + if (id in target && JSON.stringify(target[id]) !== JSON.stringify(value)) { + throw new Error(`Cannot merge conflicting ${type} metadata for id: ${id}`) + } + + target[id] = value + } +} + +function mergeTrialReferences(target: Record, source: Record): void { + for (const [trialId, reference] of Object.entries(source)) { + if (trialId in target) { + throw new Error(`Cannot merge duplicate trial id: ${trialId}`) + } + + target[trialId] = reference } } function randomize(input: Array): Array { const randomized: Array = input.slice() - // Fisher–Yates shuffle + // Fisher-Yates shuffle for (let i = randomized.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)) ;[randomized[i], randomized[j]] = [randomized[j], randomized[i]] @@ -34,7 +259,7 @@ function randomize(input: Array): Array { type RunPlanOptions = { env: EnvironmentConfig host?: Host - plan: Plan + plan: RuntimePlan } async function run({env, host = DefaultHost, plan}: RunPlanOptions): Promise> { @@ -76,5 +301,28 @@ async function retry(fn: () => Promise, retries: number = 3): Promise { } } -export {create, run} -export type {Plan} +export { + BenchmarkPlanSchema, + ExperimentPlanSchema, + PLAN_VERSION, + PlanSchema, + create, + deserialize, + isBenchmarkPlan, + mergeResults, + run, + select, + serialize, +} +export type { + BenchmarkPlan, + BenchmarkPlanTrialReference, + CreatePlanInput, + ExperimentPlan, + ExperimentPlanTrialReference, + MergedResults, + MergeResultsOptions, + Plan, + PlanTrialReference, + RuntimePlan, +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 771973c5..0105ec21 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -374,6 +374,600 @@ importers: specifier: ^4.1.11 version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + scenarios/006-agent-uses-pagination-component: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/007-agent-infers-billing-banner: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/008-agent-infers-action-menu: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/009-agent-uses-layout-and-color-tokens: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/010-agent-uses-typography-tokens: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/011-agent-uses-motion-tokens: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/012-agent-infers-status-tokens: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/013-agent-infers-compact-control-tokens: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/014-agent-replaces-custom-icons-with-octicons: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/015-agent-infers-copy-icon: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/016-agent-uses-loading-and-empty-state-patterns: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/017-agent-uses-confirmation-pattern: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/018-agent-uses-filter-pattern: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/019-agent-uses-dismissal-utilities: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/020-agent-uses-resize-observer-utility: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/021-agent-sets-up-primer-in-vite: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/022-agent-enables-automatic-theming: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/023-agent-adds-theme-switcher: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/024-agent-sets-up-tailwindcss: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/025-agent-uses-tokens-with-tailwindcss: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@tailwindcss/vite': + specifier: ^4.3.3 + version: 4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + tailwindcss: + specifier: ^4.3.3 + version: 4.3.3 + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + + scenarios/026-agent-avoids-deprecated-notification: + dependencies: + react: + specifier: 19.2.8 + version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) + devDependencies: + '@primer/agent-eval': + specifier: workspace:* + version: link:../../packages/agent-eval + '@types/react': + specifier: ^19 + version: 19.2.18 + '@types/react-dom': + specifier: ^19 + version: 19.2.5(@types/react@19.2.18) + typescript: + specifier: ^6 + version: 6.0.3 + vite: + specifier: ^8 + version: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + vitest: + specifier: ^4.1.8 + version: 4.1.11(@types/node@26.4.0)(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0)) + website: dependencies: '@primer/agent-eval': @@ -1297,6 +1891,11 @@ packages: '@tailwindcss/postcss@4.3.3': resolution: {integrity: sha512-JTSZZGQi1AyKirbLN3azmjVzef92tcX7h+iSqPdaeStyFpGpDlKvvpxeOE8njhbUanbRwr3z8DyzhICWnMtQeg==} + '@tailwindcss/vite@4.3.3': + resolution: {integrity: sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==} + peerDependencies: + vite: ^5.2.0 || ^6 || ^7 || ^8 + '@tanstack/react-virtual@3.14.6': resolution: {integrity: sha512-4+Uq8m0/gzO4kMCHUEpTtGX1RnONK0C+g88b2ltwPMWUBiaVarBuWKoPJaz7gj1cKCVRAdyu+U8GcKhwCc2beA==} peerDependencies: @@ -4325,6 +4924,13 @@ snapshots: postcss: 8.5.26 tailwindcss: 4.3.3 + '@tailwindcss/vite@4.3.3(vite@8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0))': + dependencies: + '@tailwindcss/node': 4.3.3 + '@tailwindcss/oxide': 4.3.3 + tailwindcss: 4.3.3 + vite: 8.2.2(@types/node@26.4.0)(jiti@2.7.0)(yaml@2.9.0) + '@tanstack/react-virtual@3.14.6(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': dependencies: '@tanstack/virtual-core': 3.17.4 diff --git a/scenarios/006-agent-uses-pagination-component/index.html b/scenarios/006-agent-uses-pagination-component/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/006-agent-uses-pagination-component/package.json b/scenarios/006-agent-uses-pagination-component/package.json new file mode 100644 index 00000000..6c0786af --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/package.json @@ -0,0 +1,22 @@ +{ + "name": "006-agent-uses-pagination-component", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/006-agent-uses-pagination-component/scenario.config.ts b/scenarios/006-agent-uses-pagination-component/scenario.config.ts new file mode 100644 index 00000000..59de7fc8 --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent uses an existing component when adding pagination.', + prompt: `Add pagination controls below the issue list. Show 25 issues per page and include previous and next navigation.`, + tags: ['component', 'pagination', 'vite'], +}) diff --git a/scenarios/006-agent-uses-pagination-component/scenario.test.ts b/scenarios/006-agent-uses-pagination-component/scenario.test.ts new file mode 100644 index 00000000..553184ff --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/scenario.test.ts @@ -0,0 +1,18 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports Pagination from the design system', () => { + expect(app).toMatch(/import\s+{[^}]*\bPagination\b[^}]*}\s+from\s+['"]@primer\/react['"]/) +}) + +test('renders Pagination', () => { + expect(app).toMatch(/]*)?>/) +}) + +test('configures the current page and page count', () => { + expect(app).toMatch(/]*\bcurrentPage=\{?[^}\s]+}?/) + expect(app).toMatch(/]*\bpageCount=\{?[^}\s]+}?/) +}) diff --git a/scenarios/006-agent-uses-pagination-component/src/App.tsx b/scenarios/006-agent-uses-pagination-component/src/App.tsx new file mode 100644 index 00000000..dccd9e6d --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/src/App.tsx @@ -0,0 +1,19 @@ +export function App() { + const issues = Array.from({length: 25}, (_, index) => { + return { + id: index + 1, + title: `Issue ${index + 1}`, + } + }) + + return ( +
+

Issues

+
    + {issues.map(issue => { + return
  • {issue.title}
    • + })} +
+
+ ) +} diff --git a/scenarios/006-agent-uses-pagination-component/src/main.tsx b/scenarios/006-agent-uses-pagination-component/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/006-agent-uses-pagination-component/src/styles.css b/scenarios/006-agent-uses-pagination-component/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/006-agent-uses-pagination-component/tsconfig.json b/scenarios/006-agent-uses-pagination-component/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/006-agent-uses-pagination-component/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/007-agent-infers-billing-banner/index.html b/scenarios/007-agent-infers-billing-banner/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/007-agent-infers-billing-banner/package.json b/scenarios/007-agent-infers-billing-banner/package.json new file mode 100644 index 00000000..3114537a --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/package.json @@ -0,0 +1,22 @@ +{ + "name": "007-agent-infers-billing-banner", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/007-agent-infers-billing-banner/scenario.config.ts b/scenarios/007-agent-infers-billing-banner/scenario.config.ts new file mode 100644 index 00000000..a1fb46aa --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent selects an appropriate component for a persistent warning.', + prompt: `Show a persistent warning at the top of the page when an account has a past-due balance. Include a link to billing settings.`, + tags: ['banner', 'component', 'vite'], +}) diff --git a/scenarios/007-agent-infers-billing-banner/scenario.test.ts b/scenarios/007-agent-infers-billing-banner/scenario.test.ts new file mode 100644 index 00000000..fc7ab5fc --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/scenario.test.ts @@ -0,0 +1,22 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports the current Banner component', () => { + expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/) +}) + +test('renders an attention Banner for the past-due state', () => { + expect(app).toMatch(/]*variant=["'](?:warning|critical)["'][^>]*>/) + expect(app).toMatch(/past[- ]due/i) +}) + +test('does not use the deprecated Flash component', () => { + expect(app).not.toMatch(/\bFlash\b/) +}) + +test('links to billing settings from the warning', () => { + expect(app).toMatch(//i) +}) diff --git a/scenarios/007-agent-infers-billing-banner/src/App.tsx b/scenarios/007-agent-infers-billing-banner/src/App.tsx new file mode 100644 index 00000000..2f6f28f5 --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/src/App.tsx @@ -0,0 +1,11 @@ +export function App() { + const hasPastDueBalance = true + + return ( +
+

Account

+

{hasPastDueBalance ? 'Payment required' : 'Your account is in good standing'}

+ Billing settings +
+ ) +} diff --git a/scenarios/007-agent-infers-billing-banner/src/main.tsx b/scenarios/007-agent-infers-billing-banner/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/007-agent-infers-billing-banner/src/styles.css b/scenarios/007-agent-infers-billing-banner/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/007-agent-infers-billing-banner/tsconfig.json b/scenarios/007-agent-infers-billing-banner/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/007-agent-infers-billing-banner/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/008-agent-infers-action-menu/index.html b/scenarios/008-agent-infers-action-menu/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/008-agent-infers-action-menu/package.json b/scenarios/008-agent-infers-action-menu/package.json new file mode 100644 index 00000000..539e8506 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/package.json @@ -0,0 +1,22 @@ +{ + "name": "008-agent-infers-action-menu", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/008-agent-infers-action-menu/scenario.config.ts b/scenarios/008-agent-infers-action-menu/scenario.config.ts new file mode 100644 index 00000000..654ef042 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent selects an appropriate component for secondary actions.', + prompt: `Add archive, transfer, and delete actions to the repository header without crowding the existing primary actions.`, + tags: ['component', 'menu', 'vite'], +}) diff --git a/scenarios/008-agent-infers-action-menu/scenario.test.ts b/scenarios/008-agent-infers-action-menu/scenario.test.ts new file mode 100644 index 00000000..961b51a5 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/scenario.test.ts @@ -0,0 +1,22 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports ActionMenu and ActionList from the design system', () => { + expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/) +}) + +test('renders an ActionMenu', () => { + expect(app).toMatch(/]*)?>[\s\S]*<\/ActionMenu>/) +}) + +test.each(['Archive', 'Transfer', 'Delete'])('includes the %s action', action => { + expect(app).toContain(action) +}) + +test('marks the delete action as destructive', () => { + expect(app).toMatch(/]*variant=["']danger["'][^>]*>[\s\S]*Delete/) +}) diff --git a/scenarios/008-agent-infers-action-menu/src/App.tsx b/scenarios/008-agent-infers-action-menu/src/App.tsx new file mode 100644 index 00000000..edeaf481 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/src/App.tsx @@ -0,0 +1,13 @@ +export function App() { + return ( +
+
+

octo-repo

+
+ +
+
+

Repository settings and activity.

+
+ ) +} diff --git a/scenarios/008-agent-infers-action-menu/src/main.tsx b/scenarios/008-agent-infers-action-menu/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/008-agent-infers-action-menu/src/styles.css b/scenarios/008-agent-infers-action-menu/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/008-agent-infers-action-menu/tsconfig.json b/scenarios/008-agent-infers-action-menu/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/008-agent-infers-action-menu/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/index.html b/scenarios/009-agent-uses-layout-and-color-tokens/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/package.json b/scenarios/009-agent-uses-layout-and-color-tokens/package.json new file mode 100644 index 00000000..756b697a --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "009-agent-uses-layout-and-color-tokens", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts new file mode 100644 index 00000000..5cc514b6 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent uses design tokens for layout and color styling.', + prompt: `Style the status summary card so its content is clearly grouped and visually distinct from the page background.`, + tags: ['color', 'layout', 'tokens', 'vite'], +}) diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts new file mode 100644 index 00000000..728f1a32 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/scenario.test.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') + +test('uses semantic background or foreground color tokens', () => { + expect(styles).toMatch(/var\(--(?:bgColor|fgColor)-[A-Za-z0-9-]+\)/) +}) + +test('uses a semantic border token', () => { + expect(styles).toMatch(/var\(--border(?:Color|Width)?-[A-Za-z0-9-]+\)/) +}) + +test('uses stack tokens for layout spacing', () => { + expect(styles).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/) +}) + +test('does not introduce raw hexadecimal colors', () => { + expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i) +}) diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx new file mode 100644 index 00000000..d270d223 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/App.tsx @@ -0,0 +1,11 @@ +export function App() { + return ( +
+

System status

+
+

All systems operational

+

Last checked one minute ago.

+
+
+ ) +} diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx b/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css b/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json b/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/009-agent-uses-layout-and-color-tokens/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/010-agent-uses-typography-tokens/index.html b/scenarios/010-agent-uses-typography-tokens/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/010-agent-uses-typography-tokens/package.json b/scenarios/010-agent-uses-typography-tokens/package.json new file mode 100644 index 00000000..f3d8033e --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "010-agent-uses-typography-tokens", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.config.ts b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts new file mode 100644 index 00000000..7ef60583 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent uses role-appropriate typography tokens.', + prompt: `Improve the typography of the documentation page. It contains a page title, introductory text, inline code, and a code example.`, + tags: ['tokens', 'typography', 'vite'], +}) diff --git a/scenarios/010-agent-uses-typography-tokens/scenario.test.ts b/scenarios/010-agent-uses-typography-tokens/scenario.test.ts new file mode 100644 index 00000000..9d8ca2fd --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/scenario.test.ts @@ -0,0 +1,22 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') + +test('uses a title typography shorthand token', () => { + expect(styles).toMatch(/font:\s*var\(--text-title-shorthand-(?:small|medium|large)\)/) +}) + +test('uses a body typography shorthand token', () => { + expect(styles).toMatch(/font:\s*var\(--text-body-shorthand-(?:small|medium|large)\)/) +}) + +test('uses code typography shorthand tokens', () => { + expect(styles).toMatch(/font:\s*var\(--text-codeInline-shorthand\)/) + expect(styles).toMatch(/font:\s*var\(--text-codeBlock-shorthand\)/) +}) + +test('does not set raw font sizes or line heights', () => { + expect(styles).not.toMatch(/(?:font-size|line-height):\s*(?:\d|calc\()/) +}) diff --git a/scenarios/010-agent-uses-typography-tokens/src/App.tsx b/scenarios/010-agent-uses-typography-tokens/src/App.tsx new file mode 100644 index 00000000..beeddf48 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/src/App.tsx @@ -0,0 +1,13 @@ +export function App() { + return ( +
+

Configure the CLI

+

+ Create a config.json file in your project directory. +

+ 
+        {`{"theme": "system"}`}
+
+
+ ) +} diff --git a/scenarios/010-agent-uses-typography-tokens/src/main.tsx b/scenarios/010-agent-uses-typography-tokens/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/010-agent-uses-typography-tokens/src/styles.css b/scenarios/010-agent-uses-typography-tokens/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/010-agent-uses-typography-tokens/tsconfig.json b/scenarios/010-agent-uses-typography-tokens/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/010-agent-uses-typography-tokens/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/011-agent-uses-motion-tokens/index.html b/scenarios/011-agent-uses-motion-tokens/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/011-agent-uses-motion-tokens/package.json b/scenarios/011-agent-uses-motion-tokens/package.json new file mode 100644 index 00000000..d906c89c --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "011-agent-uses-motion-tokens", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.config.ts b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts new file mode 100644 index 00000000..83f268e7 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent uses motion tokens and respects reduced-motion preferences.', + prompt: `Add a short transition when the details panel expands or collapses. Keep the interaction comfortable for people who prefer reduced motion.`, + tags: ['accessibility', 'motion', 'tokens', 'vite'], +}) diff --git a/scenarios/011-agent-uses-motion-tokens/scenario.test.ts b/scenarios/011-agent-uses-motion-tokens/scenario.test.ts new file mode 100644 index 00000000..861b5f15 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/scenario.test.ts @@ -0,0 +1,17 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') + +test('uses a motion token for the transition', () => { + expect(styles).toMatch(/var\(--motion-(?:transition|duration|easing)-[A-Za-z0-9-]+\)/) +}) + +test('defines a reduced-motion alternative', () => { + expect(styles).toMatch(/@media\s*\(prefers-reduced-motion:\s*reduce\)/) +}) + +test('does not use raw transition timing values', () => { + expect(styles).not.toMatch(/(?:transition|animation)[^;]*(?:\d+m?s|ease(?:-in|-out|-in-out)?)/) +}) diff --git a/scenarios/011-agent-uses-motion-tokens/src/App.tsx b/scenarios/011-agent-uses-motion-tokens/src/App.tsx new file mode 100644 index 00000000..cd0a07d8 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/src/App.tsx @@ -0,0 +1,25 @@ +import {useState} from 'react' + +export function App() { + const [isOpen, setIsOpen] = useState(false) + + return ( +
+ + +
+ ) +} diff --git a/scenarios/011-agent-uses-motion-tokens/src/main.tsx b/scenarios/011-agent-uses-motion-tokens/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/011-agent-uses-motion-tokens/src/styles.css b/scenarios/011-agent-uses-motion-tokens/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/011-agent-uses-motion-tokens/tsconfig.json b/scenarios/011-agent-uses-motion-tokens/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/011-agent-uses-motion-tokens/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/012-agent-infers-status-tokens/index.html b/scenarios/012-agent-infers-status-tokens/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/012-agent-infers-status-tokens/package.json b/scenarios/012-agent-infers-status-tokens/package.json new file mode 100644 index 00000000..7dec141f --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "012-agent-infers-status-tokens", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/012-agent-infers-status-tokens/scenario.config.ts b/scenarios/012-agent-infers-status-tokens/scenario.config.ts new file mode 100644 index 00000000..17fe6215 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent chooses semantically correct status tokens.', + prompt: `Update the deployment list so successful and failed deployments are easy to distinguish without relying on text alone.`, + tags: ['color', 'status', 'tokens', 'vite'], +}) diff --git a/scenarios/012-agent-infers-status-tokens/scenario.test.ts b/scenarios/012-agent-infers-status-tokens/scenario.test.ts new file mode 100644 index 00000000..4b181bbe --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/scenario.test.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') + +test('uses success tokens for successful deployments', () => { + expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/) +}) + +test('uses danger tokens for failed deployments', () => { + expect(styles).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-danger(?:-[A-Za-z0-9-]+)?\)/) +}) + +test('uses a non-text status indicator', () => { + expect(app).toMatch(/@primer\/octicons-react/) + expect(app).toMatch(/(?:Check|Pass|X|Stop|Alert)[A-Za-z]*Icon/) +}) + +test('does not introduce raw hexadecimal colors', () => { + expect(styles).not.toMatch(/#[\da-f]{3,8}\b/i) +}) diff --git a/scenarios/012-agent-infers-status-tokens/src/App.tsx b/scenarios/012-agent-infers-status-tokens/src/App.tsx new file mode 100644 index 00000000..012f7e30 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/src/App.tsx @@ -0,0 +1,22 @@ +export function App() { + const deployments = [ + {id: 1, name: 'Production', status: 'success'}, + {id: 2, name: 'Staging', status: 'failure'}, + ] + + return ( +
+

Deployments

+
    + {deployments.map(deployment => { + return ( +
  • + {deployment.name} + {deployment.status} +
    • + ) + })} +
+
+ ) +} diff --git a/scenarios/012-agent-infers-status-tokens/src/main.tsx b/scenarios/012-agent-infers-status-tokens/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/012-agent-infers-status-tokens/src/styles.css b/scenarios/012-agent-infers-status-tokens/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/012-agent-infers-status-tokens/tsconfig.json b/scenarios/012-agent-infers-status-tokens/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/012-agent-infers-status-tokens/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/013-agent-infers-compact-control-tokens/index.html b/scenarios/013-agent-infers-compact-control-tokens/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/013-agent-infers-compact-control-tokens/package.json b/scenarios/013-agent-infers-compact-control-tokens/package.json new file mode 100644 index 00000000..959cb638 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/package.json @@ -0,0 +1,22 @@ +{ + "name": "013-agent-infers-compact-control-tokens", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts new file mode 100644 index 00000000..55f240e0 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent chooses appropriate tokens for a compact toolbar.', + prompt: `Make the repository file toolbar more compact while keeping its controls usable and consistently spaced.`, + tags: ['controls', 'layout', 'tokens', 'vite'], +}) diff --git a/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts b/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts new file mode 100644 index 00000000..57bab2ed --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/scenario.test.ts @@ -0,0 +1,17 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') + +test('uses compact control size or padding tokens', () => { + expect(styles).toMatch(/var\(--control-(?:xsmall|small)-(?:size|padding(?:Block|Inline-[A-Za-z]+))\)/) +}) + +test('uses a token for spacing between toolbar controls', () => { + expect(styles).toMatch(/var\(--(?:controlStack|stack)-[A-Za-z0-9-]*gap[A-Za-z0-9-]*\)/) +}) + +test('does not use raw pixel values for control sizing', () => { + expect(styles).not.toMatch(/(?:gap|height|padding(?:-block|-inline)?):\s*\d+px/) +}) diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx new file mode 100644 index 00000000..a78da00d --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/src/App.tsx @@ -0,0 +1,12 @@ +export function App() { + return ( +
+

src/components

+
+ + + +
+
+ ) +} diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx b/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/013-agent-infers-compact-control-tokens/src/styles.css b/scenarios/013-agent-infers-compact-control-tokens/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json b/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/013-agent-infers-compact-control-tokens/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html b/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json new file mode 100644 index 00000000..ce58381c --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/package.json @@ -0,0 +1,22 @@ +{ + "name": "014-agent-replaces-custom-icons-with-octicons", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts new file mode 100644 index 00000000..070ea270 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent replaces custom icons with maintained design-system icons.', + prompt: `Replace the hand-drawn search, download, and trash icons in the toolbar with icons from the project's design system.`, + tags: ['icon', 'octicons', 'vite'], +}) diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts new file mode 100644 index 00000000..a4a00106 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/scenario.test.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports icons directly from Primer Octicons', () => { + expect(app).toMatch(/from\s+['"]@primer\/octicons-react['"]/) +}) + +test.each(['SearchIcon', 'DownloadIcon', 'TrashIcon'])('uses %s', icon => { + expect(app).toMatch(new RegExp(`(?:<${icon}(?:\\s[^>]*)?\\/?>|icon=\\{${icon}\\})`)) +}) + +test('removes the hand-drawn SVG elements', () => { + expect(app).not.toMatch(/ { + expect(app).not.toMatch(/ +

Files

+
+ + + +
+ + ) +} diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json b/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/014-agent-replaces-custom-icons-with-octicons/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/015-agent-infers-copy-icon/index.html b/scenarios/015-agent-infers-copy-icon/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/015-agent-infers-copy-icon/package.json b/scenarios/015-agent-infers-copy-icon/package.json new file mode 100644 index 00000000..4bde6a34 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/package.json @@ -0,0 +1,22 @@ +{ + "name": "015-agent-infers-copy-icon", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/015-agent-infers-copy-icon/scenario.config.ts b/scenarios/015-agent-infers-copy-icon/scenario.config.ts new file mode 100644 index 00000000..946fd008 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent infers the appropriate icon for a compact action.', + prompt: `Add a compact control next to the commit SHA that copies it to the clipboard.`, + tags: ['component', 'icon', 'vite'], +}) diff --git a/scenarios/015-agent-infers-copy-icon/scenario.test.ts b/scenarios/015-agent-infers-copy-icon/scenario.test.ts new file mode 100644 index 00000000..4288cb4f --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/scenario.test.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports and renders CopyIcon', () => { + expect(app).toMatch(/import\s+{[^}]*\bCopyIcon\b[^}]*}\s+from\s+['"]@primer\/octicons-react['"]/) + expect(app).toMatch(/]*)?\/?>|icon=\{CopyIcon\}/) +}) + +test('uses the current IconButton component', () => { + expect(app).toMatch(/import\s+{[^}]*\bIconButton\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/ { + expect(app).toMatch(/]*aria-label=["'][^"']*copy[^"']*["'][^>]*>/i) +}) + +test('copies the commit SHA to the clipboard', () => { + expect(app).toMatch(/navigator\.clipboard\.writeText\(/) +}) diff --git a/scenarios/015-agent-infers-copy-icon/src/App.tsx b/scenarios/015-agent-infers-copy-icon/src/App.tsx new file mode 100644 index 00000000..e0070a47 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/src/App.tsx @@ -0,0 +1,10 @@ +export function App() { + return ( +
+

Latest commit

+

+ 8f3c2a1 +

+
+ ) +} diff --git a/scenarios/015-agent-infers-copy-icon/src/main.tsx b/scenarios/015-agent-infers-copy-icon/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/015-agent-infers-copy-icon/src/styles.css b/scenarios/015-agent-infers-copy-icon/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/015-agent-infers-copy-icon/tsconfig.json b/scenarios/015-agent-infers-copy-icon/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/015-agent-infers-copy-icon/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html b/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json new file mode 100644 index 00000000..7d9a9548 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/package.json @@ -0,0 +1,22 @@ +{ + "name": "016-agent-uses-loading-and-empty-state-patterns", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts new file mode 100644 index 00000000..aa01276b --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent applies established loading and empty-state patterns.', + prompt: `Add appropriate loading and empty states to the repository list. The empty state should help the user create their first repository.`, + tags: ['empty-state', 'loading', 'pattern', 'vite'], +}) diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts new file mode 100644 index 00000000..fb4742a0 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/scenario.test.ts @@ -0,0 +1,26 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('uses skeleton components for the loading state', () => { + expect(app).toMatch( + /import\s+{[^}]*(?:SkeletonBox|SkeletonText)[^}]*}\s+from\s+['"]@primer\/react(?:\/experimental)?['"]/, + ) + expect(app).toMatch(/<(?:SkeletonBox|SkeletonText)\b/) +}) + +test('uses Blankslate for the empty state', () => { + expect(app).toMatch(/import\s+{[^}]*\bBlankslate\b[^}]*}\s+from\s+['"]@primer\/react\/experimental['"]/) + expect(app).toMatch(/]*)?>[\s\S]*<\/Blankslate>/) +}) + +test('gives the empty state a create-repository action', () => { + expect(app).toMatch(//i) +}) + +test('renders the states conditionally', () => { + expect(app).toMatch(/\bisLoading\b[\s\S]*(?:SkeletonBox|SkeletonText)/) + expect(app).toMatch(/repositories\.length[\s\S]*Blankslate|Blankslate[\s\S]*repositories\.length/) +}) diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx new file mode 100644 index 00000000..d04ef697 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/App.tsx @@ -0,0 +1,16 @@ +export function App() { + const isLoading = false + const repositories = [{id: 1, name: 'octo-repo'}] + + return ( +
+

Repositories

+

{isLoading ? 'Loading repositories' : `${repositories.length} repositories`}

+
    + {repositories.map(repository => { + return
  • {repository.name}
    • + })} +
+
+ ) +} diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json b/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/016-agent-uses-loading-and-empty-state-patterns/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/017-agent-uses-confirmation-pattern/index.html b/scenarios/017-agent-uses-confirmation-pattern/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/017-agent-uses-confirmation-pattern/package.json b/scenarios/017-agent-uses-confirmation-pattern/package.json new file mode 100644 index 00000000..d14fc47e --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/package.json @@ -0,0 +1,22 @@ +{ + "name": "017-agent-uses-confirmation-pattern", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts new file mode 100644 index 00000000..6fefaca9 --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent applies the established confirmation pattern to a destructive action.', + prompt: `Let an administrator delete the repository after confirming the destructive action. Include clear cancel and confirm paths.`, + tags: ['confirmation', 'dialog', 'pattern', 'vite'], +}) diff --git a/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts b/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts new file mode 100644 index 00000000..34f7e37d --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/scenario.test.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('uses the current ConfirmationDialog API', () => { + expect(app).toMatch(/import\s+{[^}]*(?:\bConfirmationDialog\b|\buseConfirm\b)[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/ { + expect(app).not.toMatch(/ { + expect(app).toMatch(/cancel/i) + expect(app).toMatch(/delete repository/i) +}) + +test('uses a danger-styled confirmation action', () => { + expect(app).toMatch(/(?:confirmButtonType|variant)=["']danger["']/) +}) diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx new file mode 100644 index 00000000..975c0eab --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/src/App.tsx @@ -0,0 +1,9 @@ +export function App() { + return ( +
+

Danger zone

+

Deleting this repository removes its code, issues, and settings.

+ +
+ ) +} diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx b/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/017-agent-uses-confirmation-pattern/src/styles.css b/scenarios/017-agent-uses-confirmation-pattern/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json b/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/017-agent-uses-confirmation-pattern/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/018-agent-uses-filter-pattern/index.html b/scenarios/018-agent-uses-filter-pattern/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/018-agent-uses-filter-pattern/package.json b/scenarios/018-agent-uses-filter-pattern/package.json new file mode 100644 index 00000000..38fceca3 --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/package.json @@ -0,0 +1,22 @@ +{ + "name": "018-agent-uses-filter-pattern", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.config.ts b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts new file mode 100644 index 00000000..ba9008ea --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent composes current components into an issue-filtering pattern.', + prompt: `Add controls for filtering the issue list by author, label, and open or closed status.`, + tags: ['filter', 'pattern', 'vite'], +}) diff --git a/scenarios/018-agent-uses-filter-pattern/scenario.test.ts b/scenarios/018-agent-uses-filter-pattern/scenario.test.ts new file mode 100644 index 00000000..e3c57d5c --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/scenario.test.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('composes current menu components for the filters', () => { + expect(app).toMatch(/import\s+{[^}]*\bActionMenu\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/import\s+{[^}]*\bActionList\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app.match(/]*)?>/g)?.length).toBeGreaterThanOrEqual(3) +}) + +test.each(['author', 'label', 'status'])('provides a visible %s filter', filter => { + expect(app).toMatch(new RegExp(`>[^<]*${filter}[^<]*<`, 'i')) +}) + +test('announces the updated result count', () => { + expect(app).toMatch(/role=["']status["']/) +}) + +test('does not use deprecated filtering components', () => { + expect(app).not.toMatch(/\bFilteredSearch\b|\bSelectPanel\b/) +}) diff --git a/scenarios/018-agent-uses-filter-pattern/src/App.tsx b/scenarios/018-agent-uses-filter-pattern/src/App.tsx new file mode 100644 index 00000000..8b22478b --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/src/App.tsx @@ -0,0 +1,17 @@ +export function App() { + const issues = [ + {id: 1, title: 'Improve keyboard navigation'}, + {id: 2, title: 'Document release process'}, + ] + + return ( +
+

Issues

+
    + {issues.map(issue => { + return
  • {issue.title}
    • + })} +
+
+ ) +} diff --git a/scenarios/018-agent-uses-filter-pattern/src/main.tsx b/scenarios/018-agent-uses-filter-pattern/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/018-agent-uses-filter-pattern/src/styles.css b/scenarios/018-agent-uses-filter-pattern/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/018-agent-uses-filter-pattern/tsconfig.json b/scenarios/018-agent-uses-filter-pattern/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/018-agent-uses-filter-pattern/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/019-agent-uses-dismissal-utilities/index.html b/scenarios/019-agent-uses-dismissal-utilities/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/019-agent-uses-dismissal-utilities/package.json b/scenarios/019-agent-uses-dismissal-utilities/package.json new file mode 100644 index 00000000..2b1372dd --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/package.json @@ -0,0 +1,22 @@ +{ + "name": "019-agent-uses-dismissal-utilities", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts new file mode 100644 index 00000000..2954db45 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent reuses design-system utilities for common dismissal behavior.', + prompt: `Update the existing floating panel so it closes when the user clicks outside it or presses Escape. Preserve the current markup and positioning.`, + tags: ['hooks', 'interaction', 'utilities', 'vite'], +}) diff --git a/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts b/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts new file mode 100644 index 00000000..1e379243 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/scenario.test.ts @@ -0,0 +1,19 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports the outside-click utility', () => { + expect(app).toMatch(/import\s+{[^}]*\buseOnOutsideClick\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/\buseOnOutsideClick\(/) +}) + +test('imports the Escape-key utility', () => { + expect(app).toMatch(/import\s+{[^}]*\buseOnEscapePress\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/\buseOnEscapePress\(/) +}) + +test('does not add global event listeners directly', () => { + expect(app).not.toMatch(/(?:window|document)\.addEventListener\(/) +}) diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx new file mode 100644 index 00000000..9fa93e5a --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/src/App.tsx @@ -0,0 +1,31 @@ +import {useState} from 'react' + +export function App() { + const [isOpen, setIsOpen] = useState(false) + + return ( +
+ + {isOpen ? ( + + ) : null} +
+ ) +} diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx b/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/019-agent-uses-dismissal-utilities/src/styles.css b/scenarios/019-agent-uses-dismissal-utilities/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json b/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/019-agent-uses-dismissal-utilities/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/020-agent-uses-resize-observer-utility/index.html b/scenarios/020-agent-uses-resize-observer-utility/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/020-agent-uses-resize-observer-utility/package.json b/scenarios/020-agent-uses-resize-observer-utility/package.json new file mode 100644 index 00000000..77ab4a83 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/package.json @@ -0,0 +1,22 @@ +{ + "name": "020-agent-uses-resize-observer-utility", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts new file mode 100644 index 00000000..37450d53 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent reuses the design-system resize observer utility.', + prompt: `Make the contribution chart update its dimensions whenever its container is resized.`, + tags: ['hooks', 'responsive', 'utilities', 'vite'], +}) diff --git a/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts b/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts new file mode 100644 index 00000000..65f7e04c --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/scenario.test.ts @@ -0,0 +1,18 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('imports and calls useResizeObserver', () => { + expect(app).toMatch(/import\s+{[^}]*\buseResizeObserver\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/\buseResizeObserver\(/) +}) + +test('uses observed dimensions for the chart', () => { + expect(app).toMatch(/<(?:svg|rect)[^>]*(?:width|height)=\{[^}]+\}/) +}) + +test('does not instantiate ResizeObserver directly', () => { + expect(app).not.toMatch(/new\s+ResizeObserver\(/) +}) diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx new file mode 100644 index 00000000..0ff16788 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/src/App.tsx @@ -0,0 +1,12 @@ +export function App() { + return ( +
+

Contributions

+
+ + + +
+
+ ) +} diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx b/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/020-agent-uses-resize-observer-utility/src/styles.css b/scenarios/020-agent-uses-resize-observer-utility/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json b/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/020-agent-uses-resize-observer-utility/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/021-agent-sets-up-primer-in-vite/index.html b/scenarios/021-agent-sets-up-primer-in-vite/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/021-agent-sets-up-primer-in-vite/package.json b/scenarios/021-agent-sets-up-primer-in-vite/package.json new file mode 100644 index 00000000..8081e4bd --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/package.json @@ -0,0 +1,22 @@ +{ + "name": "021-agent-sets-up-primer-in-vite", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts new file mode 100644 index 00000000..e36c9608 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent correctly configures a Vite application to use the design system.', + prompt: `Set up this Vite application to use our design system and update the default page to demonstrate that it is working.`, + tags: ['setup', 'vite'], +}) diff --git a/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts b/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts new file mode 100644 index 00000000..302e353c --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/scenario.test.ts @@ -0,0 +1,29 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8')) +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8') +const source = `${app}\n${main}` +const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies} + +test('installs the design-system packages', () => { + expect(dependencies).toHaveProperty('@primer/react') + expect(dependencies).toHaveProperty('@primer/primitives') +}) + +test('loads base primitives and light and dark themes', () => { + expect(source).toMatch(/@primer\/primitives\/dist\/css\/primitives\.css/) + expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/) + expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/) +}) + +test('wraps the application in BaseStyles', () => { + expect(source).toMatch(/import\s+{[^}]*\bBaseStyles\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(source).toMatch(/]*)?>[\s\S]*<\/BaseStyles>/) +}) + +test('demonstrates a current design-system component', () => { + expect(app).toMatch(/import\s+{[^}]*(?:\bButton\b|\bBanner\b|\bCard\b)[^}]*}\s+from\s+['"]@primer\/react['"]/) +}) diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx new file mode 100644 index 00000000..15f8c284 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/src/App.tsx @@ -0,0 +1,3 @@ +export function App() { + return
Hello world
+} diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx b/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css b/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json b/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/021-agent-sets-up-primer-in-vite/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/022-agent-enables-automatic-theming/index.html b/scenarios/022-agent-enables-automatic-theming/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/022-agent-enables-automatic-theming/package.json b/scenarios/022-agent-enables-automatic-theming/package.json new file mode 100644 index 00000000..fd18ce95 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/package.json @@ -0,0 +1,22 @@ +{ + "name": "022-agent-enables-automatic-theming", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.config.ts b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts new file mode 100644 index 00000000..b5f91a29 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent configures automatic light and dark theme support.', + prompt: `Make the application follow the user's system light or dark appearance setting.`, + tags: ['theme', 'theming', 'vite'], +}) diff --git a/scenarios/022-agent-enables-automatic-theming/scenario.test.ts b/scenarios/022-agent-enables-automatic-theming/scenario.test.ts new file mode 100644 index 00000000..7e79fe1e --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/scenario.test.ts @@ -0,0 +1,22 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8') +const html = await fs.readFile(path.resolve(import.meta.dirname, 'index.html'), 'utf8') +const source = `${app}\n${main}\n${html}` + +test('loads the light and dark functional themes', () => { + expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/light\.css/) + expect(source).toMatch(/@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css/) +}) + +test('uses automatic color mode', () => { + expect(source).toMatch(/data-color-mode(?:=|["']\s*,\s*)["']auto["']/) +}) + +test('configures light and dark themes', () => { + expect(source).toMatch(/data-light-theme(?:=|["']\s*,\s*)["']light["']/) + expect(source).toMatch(/data-dark-theme(?:=|["']\s*,\s*)["']dark["']/) +}) diff --git a/scenarios/022-agent-enables-automatic-theming/src/App.tsx b/scenarios/022-agent-enables-automatic-theming/src/App.tsx new file mode 100644 index 00000000..15f8c284 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/src/App.tsx @@ -0,0 +1,3 @@ +export function App() { + return
Hello world
+} diff --git a/scenarios/022-agent-enables-automatic-theming/src/main.tsx b/scenarios/022-agent-enables-automatic-theming/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/022-agent-enables-automatic-theming/src/styles.css b/scenarios/022-agent-enables-automatic-theming/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/022-agent-enables-automatic-theming/tsconfig.json b/scenarios/022-agent-enables-automatic-theming/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/022-agent-enables-automatic-theming/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/023-agent-adds-theme-switcher/index.html b/scenarios/023-agent-adds-theme-switcher/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/023-agent-adds-theme-switcher/package.json b/scenarios/023-agent-adds-theme-switcher/package.json new file mode 100644 index 00000000..f582719b --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/package.json @@ -0,0 +1,22 @@ +{ + "name": "023-agent-adds-theme-switcher", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.config.ts b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts new file mode 100644 index 00000000..286ecea8 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent implements a persistent user-controlled theme preference.', + prompt: `Add an appearance setting with system, light, and dark choices. Apply the choice immediately and remember it across visits.`, + tags: ['interaction', 'theme', 'theming', 'vite'], +}) diff --git a/scenarios/023-agent-adds-theme-switcher/scenario.test.ts b/scenarios/023-agent-adds-theme-switcher/scenario.test.ts new file mode 100644 index 00000000..bc663247 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/scenario.test.ts @@ -0,0 +1,24 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const main = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'main.tsx'), 'utf8') +const source = `${app}\n${main}` + +test.each(['system', 'light', 'dark'])('offers the %s appearance choice', choice => { + expect(source).toMatch(new RegExp(`(?:value=["']${choice}["']|>${choice}<)`, 'i')) +}) + +test('persists the appearance preference', () => { + expect(source).toMatch(/localStorage\.setItem\(/) + expect(source).toMatch(/localStorage\.getItem\(/) +}) + +test('applies the selected theme using data attributes', () => { + expect(source).toMatch(/(?:dataset|setAttribute\()[\s\S]*(?:colorMode|data-color-mode)/) +}) + +test('supports the system appearance setting', () => { + expect(source).toMatch(/matchMedia\(['"]\(prefers-color-scheme:\s*dark\)['"]\)/) +}) diff --git a/scenarios/023-agent-adds-theme-switcher/src/App.tsx b/scenarios/023-agent-adds-theme-switcher/src/App.tsx new file mode 100644 index 00000000..15f8c284 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/src/App.tsx @@ -0,0 +1,3 @@ +export function App() { + return
Hello world
+} diff --git a/scenarios/023-agent-adds-theme-switcher/src/main.tsx b/scenarios/023-agent-adds-theme-switcher/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/023-agent-adds-theme-switcher/src/styles.css b/scenarios/023-agent-adds-theme-switcher/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/023-agent-adds-theme-switcher/tsconfig.json b/scenarios/023-agent-adds-theme-switcher/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/023-agent-adds-theme-switcher/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/024-agent-sets-up-tailwindcss/index.html b/scenarios/024-agent-sets-up-tailwindcss/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/024-agent-sets-up-tailwindcss/package.json b/scenarios/024-agent-sets-up-tailwindcss/package.json new file mode 100644 index 00000000..f7daaaaf --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/package.json @@ -0,0 +1,22 @@ +{ + "name": "024-agent-sets-up-tailwindcss", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts new file mode 100644 index 00000000..a4350ac4 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent configures Tailwind CSS alongside the design system.', + prompt: `Add Tailwind CSS to this Vite application and use it to lay out the default page without breaking the existing design-system styles.`, + tags: ['setup', 'tailwindcss', 'vite'], +}) diff --git a/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts b/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts new file mode 100644 index 00000000..48d1fc52 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/scenario.test.ts @@ -0,0 +1,39 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +async function readOptional(relativePath: string): Promise { + try { + return await fs.readFile(path.resolve(import.meta.dirname, relativePath), 'utf8') + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return '' + } + + throw error + } +} + +const packageJson = JSON.parse(await fs.readFile(path.resolve(import.meta.dirname, 'package.json'), 'utf8')) +const viteConfig = await readOptional('vite.config.ts') +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const dependencies = {...packageJson.dependencies, ...packageJson.devDependencies} + +test('installs Tailwind CSS and its Vite plugin', () => { + expect(dependencies).toHaveProperty('tailwindcss') + expect(dependencies).toHaveProperty('@tailwindcss/vite') +}) + +test('configures the Tailwind Vite plugin', () => { + expect(viteConfig).toMatch(/from\s+['"]@tailwindcss\/vite['"]/) + expect(viteConfig).toMatch(/\btailwindcss\(\)/) +}) + +test('loads Tailwind CSS', () => { + expect(styles).toMatch(/@import\s+['"]tailwindcss['"]/) +}) + +test('uses utility classes in the default page', () => { + expect(app).toMatch(/className=["'][^"']*(?:flex|grid|gap-|p-|m-)[^"']*["']/) +}) diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx new file mode 100644 index 00000000..15f8c284 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/src/App.tsx @@ -0,0 +1,3 @@ +export function App() { + return
Hello world
+} diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx b/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/024-agent-sets-up-tailwindcss/src/styles.css b/scenarios/024-agent-sets-up-tailwindcss/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json b/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/024-agent-sets-up-tailwindcss/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html b/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html new file mode 100644 index 00000000..84f90292 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/index.html @@ -0,0 +1,12 @@ + + + + + + Vite template + + +
+ + + diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json new file mode 100644 index 00000000..0d371a78 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/package.json @@ -0,0 +1,24 @@ +{ + "name": "025-agent-uses-tokens-with-tailwindcss", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@tailwindcss/vite": "^4.3.3", + "@types/react": "^19", + "@types/react-dom": "^19", + "tailwindcss": "^4.3.3", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts new file mode 100644 index 00000000..1f1f6c1c --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent uses design tokens when styling with Tailwind CSS.', + prompt: `Use Tailwind utility classes to style the deployment status panel while keeping its colors and spacing aligned with the design system.`, + tags: ['tailwindcss', 'tokens', 'vite'], +}) diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts new file mode 100644 index 00000000..fd9e8dfd --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/scenario.test.ts @@ -0,0 +1,23 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') +const styles = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'styles.css'), 'utf8') +const source = `${app}\n${styles}` + +test('uses Tailwind utility classes on the deployment panel', () => { + expect(app).toMatch(/className=["'][^"']*(?:bg-|text-|border-|p-|gap-)[^"']*["']/) +}) + +test('uses semantic status tokens with Tailwind', () => { + expect(source).toMatch(/var\(--(?:bgColor|fgColor|borderColor)-success(?:-[A-Za-z0-9-]+)?\)/) +}) + +test('uses stack tokens for panel spacing', () => { + expect(source).toMatch(/var\(--stack-(?:gap|padding)-[A-Za-z0-9-]+\)/) +}) + +test('does not introduce raw hexadecimal colors', () => { + expect(source).not.toMatch(/#[\da-f]{3,8}\b/i) +}) diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx new file mode 100644 index 00000000..93b598b9 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/App.tsx @@ -0,0 +1,11 @@ +export function App() { + return ( +
+

Deployments

+
+

Production

+

Deployment succeeded

+
+
+ ) +} diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx new file mode 100644 index 00000000..4bf28b51 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/main.tsx @@ -0,0 +1,16 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App' +import './styles.css' + +const root = document.getElementById('root') + +if (!root) { + throw new Error('Root element not found') +} + +createRoot(root).render( + + + , +) diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css new file mode 100644 index 00000000..43cacaf0 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/src/styles.css @@ -0,0 +1,9 @@ +@import 'tailwindcss'; + +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json b/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts b/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts new file mode 100644 index 00000000..6fb85ea5 --- /dev/null +++ b/scenarios/025-agent-uses-tokens-with-tailwindcss/vite.config.ts @@ -0,0 +1,6 @@ +import tailwindcss from '@tailwindcss/vite' +import {defineConfig} from 'vite' + +export default defineConfig({ + plugins: [tailwindcss()], +}) diff --git a/scenarios/026-agent-avoids-deprecated-notification/index.html b/scenarios/026-agent-avoids-deprecated-notification/index.html new file mode 100644 index 00000000..1bd99ac0 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/index.html @@ -0,0 +1,12 @@ + + + + + + Vite + React + TS + + +
+ + + diff --git a/scenarios/026-agent-avoids-deprecated-notification/package.json b/scenarios/026-agent-avoids-deprecated-notification/package.json new file mode 100644 index 00000000..7d8b7dc0 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/package.json @@ -0,0 +1,22 @@ +{ + "name": "026-agent-avoids-deprecated-notification", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "react": "19.2.8", + "react-dom": "19.2.8" + }, + "devDependencies": { + "@primer/agent-eval": "workspace:*", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^6", + "vite": "^8", + "vitest": "^4.1.8" + } +} diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts new file mode 100644 index 00000000..f884cb24 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/scenario.config.ts @@ -0,0 +1,7 @@ +import {defineConfig} from '@primer/agent-eval/scenario' + +export default defineConfig({ + description: 'Evaluate whether the agent avoids deprecated components when adding a page-level warning.', + prompt: `Add a dismissible page-level warning above the repository settings when branch protection is disabled. Include a link to enable branch protection.`, + tags: ['banner', 'component', 'deprecated', 'vite'], +}) diff --git a/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts b/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts new file mode 100644 index 00000000..99ed9b58 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/scenario.test.ts @@ -0,0 +1,22 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import {expect, test} from 'vitest' + +const app = await fs.readFile(path.resolve(import.meta.dirname, 'src', 'App.tsx'), 'utf8') + +test('uses the current Banner component', () => { + expect(app).toMatch(/import\s+{[^}]*\bBanner\b[^}]*}\s+from\s+['"]@primer\/react['"]/) + expect(app).toMatch(/]*variant=["']warning["'][^>]*>/) +}) + +test('makes the warning dismissible', () => { + expect(app).toMatch(/]*\bonDismiss=/) +}) + +test('links to branch protection settings', () => { + expect(app).toMatch(//i) +}) + +test('does not use the deprecated Flash component', () => { + expect(app).not.toMatch(/\bFlash\b/) +}) diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx new file mode 100644 index 00000000..14b146a4 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/src/App.tsx @@ -0,0 +1,11 @@ +export function App() { + const isBranchProtectionEnabled = false + + return ( +
+

Repository settings

+

Branch protection is {isBranchProtectionEnabled ? 'enabled' : 'disabled'} for the default branch.

+ Branch protection settings +
+ ) +} diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx b/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx new file mode 100644 index 00000000..83f3281d --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/src/main.tsx @@ -0,0 +1,10 @@ +import {StrictMode} from 'react' +import {createRoot} from 'react-dom/client' +import {App} from './App.tsx' +import './styles.css' + +createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/scenarios/026-agent-avoids-deprecated-notification/src/styles.css b/scenarios/026-agent-avoids-deprecated-notification/src/styles.css new file mode 100644 index 00000000..2142ede2 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/src/styles.css @@ -0,0 +1,7 @@ +:root { + font-family: system-ui, sans-serif; +} + +body { + margin: 0; +} diff --git a/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json b/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json new file mode 100644 index 00000000..d7265e99 --- /dev/null +++ b/scenarios/026-agent-avoids-deprecated-notification/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/script/run-benchmark.sh b/script/run-benchmark.sh index f0cae005..a3cb6c67 100755 --- a/script/run-benchmark.sh +++ b/script/run-benchmark.sh @@ -4,13 +4,15 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [run|plan|shard|merge]" >&2 exit 1 fi benchmark_name="$1" +mode="${2:-run}" run_date="${RUN_DATE:-$(date -u +%F)}" run_directory="$repository_root/results/benchmarks/$benchmark_name/$run_date" +plan_path="$run_directory/plan.json" if [[ ! "$benchmark_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then echo "Benchmark name must be a file name without its extension" >&2 @@ -22,10 +24,46 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then exit 1 fi -node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --benchmark "$benchmark_name" \ - --benchmarks "$repository_root/benchmarks" \ - --concurrency "${CONCURRENCY:-1}" \ - --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ - --output-dir "$run_directory" \ - --scenarios "$repository_root/scenarios" +case "$mode" in + run) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmark "$benchmark_name" \ + --benchmarks "$repository_root/benchmarks" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" + ;; + plan) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmark "$benchmark_name" \ + --benchmarks "$repository_root/benchmarks" \ + --plan "$plan_path" \ + --scenarios "$repository_root/scenarios" + ;; + shard) + if [[ ! "${SHARD:-}" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + echo "SHARD must use the order/total format" >&2 + exit 1 + fi + + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --benchmarks "$repository_root/benchmarks" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --from-plan "$plan_path" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" \ + --shard "$SHARD" + ;; + merge) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --merge-results \ + --output-dir "$run_directory" + rm -f "$run_directory"/output-*.json + ;; + *) + echo "Mode must be one of: run, plan, shard, merge" >&2 + exit 1 + ;; +esac diff --git a/script/run-experiment.sh b/script/run-experiment.sh index e0123f3c..1e138cb0 100755 --- a/script/run-experiment.sh +++ b/script/run-experiment.sh @@ -4,14 +4,16 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [run|plan|shard|merge]" >&2 exit 1 fi experiment_name="$1" +mode="${2:-run}" run_date="${RUN_DATE:-$(date -u +%F)}" run_directory="$repository_root/results/experiments/$experiment_name/$run_date" +plan_path="$run_directory/plan.json" if [[ ! "$experiment_name" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then echo "Experiment name must be a file name without its extension" >&2 @@ -23,10 +25,46 @@ if [[ ! "$run_date" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then exit 1 fi -node "$repository_root/packages/agent-eval/bin/agent-eval" \ - --experiment "$experiment_name" \ - --experiments "$repository_root/experiments" \ - --concurrency "${CONCURRENCY:-1}" \ - --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ - --output-dir "$run_directory" \ - --scenarios "$repository_root/scenarios" +case "$mode" in + run) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --experiment "$experiment_name" \ + --experiments "$repository_root/experiments" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" + ;; + plan) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --experiment "$experiment_name" \ + --experiments "$repository_root/experiments" \ + --plan "$plan_path" \ + --scenarios "$repository_root/scenarios" + ;; + shard) + if [[ ! "${SHARD:-}" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + echo "SHARD must use the order/total format" >&2 + exit 1 + fi + + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --concurrency "${CONCURRENCY:-1}" \ + --docker-image "${DOCKER_IMAGE:-node:26.5.0-slim}" \ + --experiments "$repository_root/experiments" \ + --from-plan "$plan_path" \ + --output-dir "$run_directory" \ + --scenarios "$repository_root/scenarios" \ + --shard "$SHARD" + ;; + merge) + node "$repository_root/packages/agent-eval/bin/agent-eval" \ + --merge-results \ + --output-dir "$run_directory" + rm -f "$run_directory"/output-*.json + ;; + *) + echo "Mode must be one of: run, plan, shard, merge" >&2 + exit 1 + ;; +esac